Example data notebook
Contents
Example data notebook¶
Content in this page can be created by using Jupyter Notebooks. This means that you can write content very easily, embed images, HTML, etc into your posts!
See also
Check the documentation on the syntax in here.
Writing content¶
The idea for this file is to serve as a data analysis recipe to reproduce your analysis. With both, code, outputs, and data, we can ensure that the findings are actually FAIR. Let’s look at some examples!
Load data, plot, process, and export¶
This example will load data from the SmartCitizen API, do some plots, and export it in CSV format
# Import the Test data structure
from scdata import Test
We can set the framework to output verbose information, and set the rendering framework as jupyterlab
from scdata._config import config
# Output levels:
# 'QUIET': nothing,
# 'NORMAL': warn, err
# 'DEBUG': info, warn, err, success
config._out_level='NORMAL'
# This defaults to script'. If using it in jupyterlab, sets the plot renderers accordingly
config._framework='jupyterlab'
We can put the data in a test. A test is simply a handy way to group devices and process them
test = Test('EXAMPLE_MINKE')
[2022-05-30 12:25:46] - [INFO] Possible tests found
[2022-05-30 12:25:46] - [INFO] 4 --- 2022_05_EXAMPLE_MINKE
[2022-05-30 12:25:46] - [INFO] // --- \\
[2022-05-30 12:25:46] - [INFO] Test full name, 2022_05_EXAMPLE_MINKE
Before adding devices to the test, we import the Device object:
from scdata import Device
Load SC API data¶
# Add as many devices as needed. See understanding blueprints below for more info
test.add_device(Device(blueprint = 'sc_21_station_module', descriptor = {'source': 'api',
'id': '14638',
'min_date': '2021-10-15',
'max_date': '2021-10-30'
}
)
)
[2022-05-30 12:25:46] - [SUCCESS] Device 14638 initialised
# Add as many devices as needed. See understanding blueprints below for more info
test.add_device(Device(blueprint = 'sck_21', descriptor = {'source': 'api',
'id': '14735',
'max_date': '2022-02-15',
}
)
)
[2022-05-30 12:25:47] - [SUCCESS] Hardware described in url is valid
[2022-05-30 12:25:47] - [WARNING] Skipping blueprint as it was defined in device constructor
[2022-05-30 12:25:47] - [SUCCESS] Device 14735 initialised
The blueprint urls can be found at ~/.config/scdata/config.yaml or here in:
config.blueprints
{'base': {'clean_na': None,
'documentation': None,
'post_info': None,
'frequency': None,
'id': None,
'timezone': None,
'max_date': None,
'min_date': None,
'source': None,
'version': None},
'csic_station': {'clean_na': None,
'documentation': 'https://analisi.transparenciacatalunya.cat/',
'frequency': None,
'id': None,
'timezone': None,
'max_date': None,
'min_date': None,
'processed_data_file': None,
'raw_data_file': None,
'sensors': {'C6H6': {'id': 'C6H6', 'units': 'ug/m3'},
'CO': {'id': 'CO', 'units': 'mg/m3'},
'H2S': {'id': 'H2S', 'units': 'ug/m3'},
'NO': {'id': 'NO', 'units': 'ug/m3'},
'NO2': {'id': 'NO2', 'units': 'ug/m3'},
'NOX': {'id': 'NOX', 'units': 'ug/m3'},
'O3': {'id': 'O3', 'units': 'ug/m3'},
'PM_10': {'id': 'PM10', 'units': 'ug/m3'},
'PM_1': {'id': 'PM1', 'units': 'ug/m3'},
'PM_25': {'id': 'PM2.5', 'units': 'ug/m3'},
'SO2': {'id': 'SO2', 'units': 'ug/m3'}},
'source': None,
'sources': {'api': {'handler': 'DadesObertesApiDevice'},
'csv': {'header_skip': [1, 4],
'index': 'date',
'sep': ',',
'tz-aware': True}},
'version': None},
'muv_station': {'clean_na': None,
'documentation': 'https://github.com/waagsociety/air_quality_sensor_kit/tree/master/MUV%20Kit',
'frequency': None,
'id': None,
'info_data_file': None,
'timezone': None,
'max_date': None,
'min_date': None,
'processed_data_file': None,
'raw_data_file': None,
'sensors': {'GB_2A': {'id': 'no2op2', 'units': 'mV'},
'GB_2W': {'id': 'no2op1', 'units': 'mV'},
'GB_3A': {'id': 'o3op2', 'units': 'mV'},
'GB_3W': {'id': 'o3op1', 'units': 'mV'},
'HUM': {'id': 'h', 'units': '%rh'},
'NOISE_A': {'id': 'dB', 'units': 'dBA'},
'PM_10': {'id': 'p10', 'units': 'ug/m3'},
'PM_25': {'id': 'p25', 'units': 'ug/m3'},
'TEMP': {'id': 't', 'units': 'degC'}},
'source': None,
'sources': {'api': {'handler': 'MuvApiDevice'},
'csv': {'header_skip': None, 'index': 'Time'},
'sep': ',',
'tz-aware': True},
'version': None},
'sc_20_station_iscape': {'clean_na': None,
'documentation': 'https://docs.smartcitizen.me/',
'frequency': None,
'id': None,
'info_data_file': None,
'timezone': None,
'max_date': None,
'metrics': {'CO': {'desc': 'CO calculation based on simple electrode substraction',
'kwargs': {'auxiliary': 'GB_1A',
'hardware': 'alphadelta',
'id': None,
'pollutant': 'CO',
'working': 'GB_1W'},
'process': 'basic_4electrode_alg',
'units': 'ppm',
'post': False,
'id': None},
'EXT_PM_10_CLEAN': {'desc': 'PM10 calculated based on both PMS5003 PM10 inputs',
'kwargs': {'factor': 0.3,
'limits': [0, 1000],
'names': ['EXT_PM_A_10', 'EXT_PM_B_10'],
'pick': 'min',
'window_size': 5,
'window_type': None},
'process': 'merge_ts',
'units': 'ug/m3',
'post': False,
'id': None},
'EXT_PM_1_CLEAN': {'desc': 'PM1 calculated based on both PMS5003 PM1 inputs',
'kwargs': {'factor': 0.3,
'limits': [0, 1000],
'names': ['EXT_PM_A_1', 'EXT_PM_B_1'],
'pick': 'min',
'window_size': 5,
'window_type': None},
'process': 'merge_ts',
'units': 'ug/m3',
'post': False,
'id': None},
'EXT_PM_25_CLEAN': {'desc': 'PM2.5 calculated based on both PMS5003 PM2.5 inputs',
'kwargs': {'factor': 0.3,
'limits': [0, 1000],
'names': ['EXT_PM_A_25', 'EXT_PM_B_25'],
'pick': 'min',
'window_size': 5,
'window_type': None},
'process': 'merge_ts',
'units': 'ug/m3',
'post': False,
'id': None}},
'min_date': None,
'plaftorm_id': 19,
'processed_data_file': None,
'raw_data_file': None,
'sensors': {'BATT': {'desc': 'Device battery status',
'id': '10',
'units': '%'},
'EXT_PM_A_1': {'desc': 'PM1 measurement from PMS5003 A',
'id': '71',
'units': 'ug/m3'},
'EXT_PM_A_10': {'desc': 'PM10 measurement from PMS5003 A',
'id': '73',
'units': 'ug/m3'},
'EXT_PM_A_25': {'desc': 'PM2.5 measurement from PMS5003 A',
'id': '72',
'units': 'ug/m3'},
'EXT_PM_B_1': {'desc': 'PM1 measurement from PMS5003 B',
'id': '75',
'units': 'ug/m3'},
'EXT_PM_B_10': {'desc': 'PM10 measurement from PMS5003 B',
'id': '77',
'units': 'ug/m3'},
'EXT_PM_B_25': {'desc': 'PM2.5 measurement from PMS5003 B',
'id': '76',
'units': 'ug/m3'},
'EXT_TEMP': {'desc': 'Dallas External probe Temperature',
'id': '96',
'units': 'degC'},
'GB_1A': {'desc': 'Auxiliary electrode 1', 'id': '65', 'units': 'mV'},
'GB_1W': {'desc': 'Working electrode 1', 'id': '64', 'units': 'mV'},
'GB_2A': {'desc': 'Auxiliary electrode 2', 'id': '62', 'units': 'mV'},
'GB_2W': {'desc': 'Working electrode 2', 'id': '61', 'units': 'mV'},
'GB_3A': {'desc': 'Auxiliary electrode 3', 'id': '68', 'units': 'mV'},
'GB_3W': {'desc': 'Working electrode 3', 'id': '67', 'units': 'mV'},
'GB_HUM': {'desc': 'Gases board humidity (SHT31)',
'id': '80',
'units': '%rh'},
'GB_TEMP': {'desc': 'Gases board temperature (SHT31)',
'id': '79',
'units': 'degC'},
'HUM': {'desc': 'Urban board humidity (SHT31)', 'id': '56', 'units': '%rh'},
'LIGHT': {'desc': 'Urban board ambient light', 'id': '14', 'units': 'lux'},
'TEMP': {'desc': 'Urban board temperature (SHT31)',
'id': '55',
'units': 'degC'}},
'source': None,
'sources': {'api': {'handler': 'ScApiDevice'},
'csv': {'header_skip': [1, 2, 3],
'index': 'TIME',
'sep': ',',
'tz-aware': True}},
'version': None},
'sc_21_station_iscape': {'clean_na': None,
'documentation': 'https://docs.smartcitizen.me/',
'frequency': None,
'id': None,
'info_data_file': None,
'timezone': None,
'max_date': None,
'metrics': {'CCS811_ECO2_CLEAN': {'desc': 'eCO2 cleaned data',
'kwargs': {'limits': [400, 65000],
'name': 'CCS811_ECO2',
'window_size': 5,
'window_type': None},
'process': 'clean_ts',
'units': 'ppm',
'post': False,
'id': None},
'CCS811_VOCS_CLEAN': {'desc': 'Volatile Organic Compounds cleaned data',
'kwargs': {'limits': [0, 65000],
'name': 'CCS811_VOCS',
'window_size': 5,
'window_type': None},
'process': 'clean_ts',
'units': 'ppb',
'post': False,
'id': None},
'CO': {'desc': 'CO calculation based on simple electrode substraction',
'kwargs': {'auxiliary': 'GB_1A',
'hardware': 'alphadelta',
'id': None,
'pollutant': 'CO',
'working': 'GB_1W'},
'process': 'basic_4electrode_alg',
'units': 'ppm',
'post': False,
'id': None},
'EXT_PM_10_CLEAN': {'desc': 'PM10 calculated based on both PMS5003 PM10 inputs',
'kwargs': {'factor': 0.3,
'limits': [0, 1000],
'names': ['EXT_PM_A_10', 'EXT_PM_B_10'],
'pick': 'min',
'window_size': 5,
'window_type': None},
'process': 'merge_ts',
'units': 'ug/m3',
'post': False,
'id': None},
'EXT_PM_1_CLEAN': {'desc': 'PM1 calculated based on both PMS5003 PM1 inputs',
'kwargs': {'factor': 0.3,
'limits': [0, 1000],
'names': ['EXT_PM_A_1', 'EXT_PM_B_1'],
'pick': 'min',
'window_size': 5,
'window_type': None},
'process': 'merge_ts',
'units': 'ug/m3',
'post': False,
'id': None},
'EXT_PM_25_CLEAN': {'desc': 'PM2.5 calculated based on both PMS5003 PM2.5 inputs',
'kwargs': {'factor': 0.3,
'limits': [0, 1000],
'names': ['EXT_PM_A_25', 'EXT_PM_B_25'],
'pick': 'min',
'window_size': 5,
'window_type': None},
'process': 'merge_ts',
'units': 'ug/m3',
'post': False,
'id': None}},
'min_date': None,
'plaftorm_id': None,
'processed_data_file': None,
'raw_data_file': None,
'sensors': {'BATT': {'desc': 'Device battery status',
'id': '10',
'units': '%'},
'CCS811_ECO2': {'id': '112', 'units': 'ppm'},
'CCS811_VOCS': {'id': '113', 'units': 'ppm'},
'EXT_PM_A_1': {'desc': 'PM1 measurement from PMS5003 A',
'id': '71',
'units': 'ug/m3'},
'EXT_PM_A_10': {'desc': 'PM10 measurement from PMS5003 A',
'id': '73',
'units': 'ug/m3'},
'EXT_PM_A_25': {'desc': 'PM2.5 measurement from PMS5003 A',
'id': '72',
'units': 'ug/m3'},
'EXT_PM_B_1': {'desc': 'PM1 measurement from PMS5003 B',
'id': '75',
'units': 'ug/m3'},
'EXT_PM_B_10': {'desc': 'PM10 measurement from PMS5003 B',
'id': '77',
'units': 'ug/m3'},
'EXT_PM_B_25': {'desc': 'PM2.5 measurement from PMS5003 B',
'id': '76',
'units': 'ug/m3'},
'EXT_TEMP': {'desc': 'Dallas External probe Temperature',
'id': '96',
'units': 'degC'},
'GB_1A': {'desc': 'Auxiliary electrode 1', 'id': '65', 'units': 'mV'},
'GB_1W': {'desc': 'Working electrode 1', 'id': '64', 'units': 'mV'},
'GB_2A': {'desc': 'Auxiliary electrode 2', 'id': '62', 'units': 'mV'},
'GB_2W': {'desc': 'Working electrode 2', 'id': '61', 'units': 'mV'},
'GB_3A': {'desc': 'Auxiliary electrode 3', 'id': '68', 'units': 'mV'},
'GB_3W': {'desc': 'Working electrode 3', 'id': '67', 'units': 'mV'},
'GB_HUM': {'desc': 'Gases board humidity (SHT31)',
'id': '80',
'units': '%rh'},
'GB_TEMP': {'desc': 'Gases board temperature (SHT31)',
'id': '79',
'units': 'degC'},
'HUM': {'desc': 'Urban board humidity (SHT31)', 'id': '56', 'units': '%rh'},
'LIGHT': {'desc': 'Urban board ambient light', 'id': '14', 'units': 'lux'},
'NOISE_A': {'desc': 'A-scale noise SPL', 'id': '53', 'units': 'dBA'},
'TEMP': {'desc': 'Urban board temperature (SHT31)',
'id': '55',
'units': 'degC'}},
'source': None,
'sources': {'api': {'handler': 'ScApiDevice'},
'csv': {'header_skip': [1, 2, 3],
'index': 'TIME',
'sep': ',',
'tz-aware': True}},
'version': None},
'sc_21_station_module': {'clean_na': None,
'documentation': 'https://docs.smartcitizen.me/',
'frequency': None,
'id': None,
'info_data_file': None,
'timezone': None,
'max_date': None,
'metrics': {'CCS811_ECO2_CLEAN': {'desc': 'eCO2 cleaned data',
'kwargs': {'limits': [400, 65000],
'name': 'CCS811_ECO2',
'window_size': 5,
'window_type': None},
'process': 'clean_ts',
'units': 'ppm',
'post': False,
'id': None},
'CCS811_VOCS_CLEAN': {'desc': 'Volatile Organic Compounds cleaned data',
'kwargs': {'limits': [0, 65000],
'name': 'CCS811_VOCS',
'window_size': 5,
'window_type': None},
'process': 'clean_ts',
'units': 'ppb',
'post': False,
'id': None},
'EXT_PM_10_CLEAN': {'desc': 'PM10 calculated based on both PMS5003 PM10 inputs',
'kwargs': {'factor': 0.3,
'limits': [0, 1000],
'names': ['EXT_PM_A_10', 'EXT_PM_B_10'],
'pick': 'min',
'window_size': 5,
'window_type': None},
'process': 'merge_ts',
'units': 'ug/m3',
'post': True,
'id': 88},
'EXT_PM_1_CLEAN': {'desc': 'PM1 calculated based on both PMS5003 PM1 inputs',
'kwargs': {'factor': 0.3,
'limits': [0, 1000],
'names': ['EXT_PM_A_1', 'EXT_PM_B_1'],
'pick': 'min',
'window_size': 5,
'window_type': None},
'process': 'merge_ts',
'units': 'ug/m3',
'post': True,
'id': 89},
'EXT_PM_25_CLEAN': {'desc': 'PM2.5 calculated based on both PMS5003 PM2.5 inputs',
'kwargs': {'factor': 0.3,
'limits': [0, 1000],
'names': ['EXT_PM_A_25', 'EXT_PM_B_25'],
'pick': 'min',
'window_size': 5,
'window_type': None},
'process': 'merge_ts',
'units': 'ug/m3',
'post': True,
'id': 87},
'PT1000_POS': {'desc': 'PT1000 raw value',
'id': None,
'kwargs': {'channel': None},
'post': False,
'process': 'channel_names',
'units': 'V'},
'ASPT1000': {'desc': 'PT1000 temperature calculation in AFE',
'id': None,
'kwargs': {'pt1000minus': None,
'from_date': None,
'timezone': None,
'to_date': None,
'pt1000plus': None,
'afe_id': None},
'post': False,
'process': 'alphasense_pt1000',
'units': 'degC'},
'EC_SENSOR_TEMP': {'desc': 'Electrochemical sensor temperature',
'id': 'X013',
'kwargs': {'priority': 'ASPT1000'},
'post': False,
'process': 'ec_sensor_temp',
'units': 'degC'},
'CO_WE': {'desc': 'CO working electrode raw value',
'id': 'X001',
'kwargs': {'channel': None},
'post': False,
'process': 'channel_names',
'units': 'V'},
'CO_AE': {'desc': 'CO auxiliary electrode raw value',
'id': 'X002',
'kwargs': {'channel': None},
'post': False,
'process': 'channel_names',
'units': 'V'},
'NO2_WE': {'desc': 'NO2 working electrode raw value',
'id': 'X003',
'kwargs': {'channel': None},
'post': False,
'process': 'channel_names',
'units': 'V'},
'NO2_AE': {'desc': 'NO2 auxiliary electrode raw value',
'id': 'X004',
'kwargs': {'channel': None},
'post': False,
'process': 'channel_names',
'units': 'V'},
'NO_WE': {'desc': 'NO working electrode raw value',
'id': 'X005',
'kwargs': {'channel': None},
'post': False,
'process': 'channel_names',
'units': 'V'},
'NO_AE': {'desc': 'NO auxiliary electrode raw value',
'id': 'X006',
'kwargs': {'channel': None},
'post': False,
'process': 'channel_names',
'units': 'V'},
'SO2_WE': {'desc': 'SO2 working electrode raw value',
'id': 'X007',
'kwargs': {'channel': None},
'post': False,
'process': 'channel_names',
'units': 'V'},
'SO2_AE': {'desc': 'SO2 auxiliary electrode raw value',
'id': 'X008',
'kwargs': {'channel': None},
'post': False,
'process': 'channel_names',
'units': 'V'},
'H2S_WE': {'desc': 'H2S working electrode raw value',
'id': 'X009',
'kwargs': {'channel': None},
'post': False,
'process': 'channel_names',
'units': 'V'},
'H2S_AE': {'desc': 'H2S auxiliary electrode raw value',
'id': 'X010',
'kwargs': {'channel': None},
'post': False,
'process': 'channel_names',
'units': 'V'},
'OX_WE': {'desc': 'OX working electrode raw value',
'id': 'X011',
'kwargs': {'channel': None},
'post': False,
'process': 'channel_names',
'units': 'V'},
'OX_AE': {'desc': 'OX auxiliary electrode raw value',
'id': 'X012',
'kwargs': {'channel': None},
'post': False,
'process': 'channel_names',
'units': 'V'},
'CO': {'desc': 'Calculation of CO based on AAN 803-04',
'id': 152,
'kwargs': {'ae': None,
'alphasense_id': None,
'from_date': None,
'timezone': None,
't': 'EC_SENSOR_TEMP',
'to_date': None,
'we': None},
'post': True,
'process': 'alphasense_803_04',
'units': 'ppb'},
'NO2': {'desc': 'Calculation of NO2 based on AAN 803-04',
'id': 153,
'kwargs': {'ae': None,
'alphasense_id': None,
'from_date': None,
'timezone': None,
't': 'EC_SENSOR_TEMP',
'to_date': None,
'we': None},
'post': True,
'process': 'alphasense_803_04',
'units': 'ppb'},
'O3': {'desc': 'Calculation of O3 based on AAN 803-04',
'id': 157,
'kwargs': {'ae': None,
'alphasense_id': None,
'from_date': None,
'timezone': None,
't': 'EC_SENSOR_TEMP',
'to_date': None,
'we': None},
'post': True,
'process': 'alphasense_803_04',
'units': 'ppb'},
'SO2': {'desc': 'Calculation of SO2 based on AAN 803-04',
'id': 155,
'kwargs': {'ae': None,
'alphasense_id': None,
'from_date': None,
'timezone': None,
't': 'EC_SENSOR_TEMP',
'to_date': None,
'we': None,
'use_alternative': True},
'post': True,
'process': 'alphasense_803_04',
'units': 'ppb'},
'NO': {'desc': 'Calculation of NO based on AAN 803-04',
'id': 154,
'kwargs': {'ae': None,
'alphasense_id': None,
'from_date': None,
'timezone': None,
't': 'EC_SENSOR_TEMP',
'to_date': None,
'we': None},
'post': True,
'process': 'alphasense_803_04',
'units': 'ppb'},
'H2S': {'desc': 'Calculation of H2S based on AAN 803-04',
'id': 156,
'kwargs': {'ae': None,
'alphasense_id': None,
'from_date': None,
'timezone': None,
't': 'EC_SENSOR_TEMP',
'to_date': None,
'we': None},
'post': True,
'process': 'alphasense_803_04',
'units': 'ppb'}},
'min_date': None,
'platform_id': 28,
'processed_data_file': None,
'raw_data_file': None,
'sensors': {'ADC_48_0': {'desc': 'Channel 0 of ADC 0x48',
'id': 133,
'units': 'V'},
'ADC_48_1': {'desc': 'Channel 1 of ADC 0x48', 'id': 134, 'units': 'V'},
'ADC_48_2': {'desc': 'Channel 2 of ADC 0x48', 'id': 135, 'units': 'V'},
'ADC_48_3': {'desc': 'Channel 3 of ADC 0x48', 'id': 136, 'units': 'V'},
'ADC_49_0': {'desc': 'Channel 0 of ADC 0x49', 'id': 138, 'units': 'V'},
'ADC_49_1': {'desc': 'Channel 1 of ADC 0x49', 'id': 139, 'units': 'V'},
'ADC_49_2': {'desc': 'Channel 2 of ADC 0x49', 'id': 140, 'units': 'V'},
'ADC_49_3': {'desc': 'Channel 3 of ADC 0x49', 'id': 141, 'units': 'V'},
'ADC_4A_0': {'desc': 'Channel 0 of ADC 0x4A', 'id': 143, 'units': 'V'},
'ADC_4A_1': {'desc': 'Channel 1 of ADC 0x4A', 'id': 144, 'units': 'V'},
'ADC_4A_2': {'desc': 'Channel 2 of ADC 0x4A', 'id': 145, 'units': 'V'},
'ADC_4A_3': {'desc': 'Channel 3 of ADC 0x4A', 'id': 146, 'units': 'V'},
'ADC_4B_0': {'desc': 'Channel 0 of ADC 0x4B', 'id': 148, 'units': 'V'},
'ADC_4B_1': {'desc': 'Channel 1 of ADC 0x4B', 'id': 149, 'units': 'V'},
'ADC_4B_2': {'desc': 'Channel 2 of ADC 0x4B', 'id': 150, 'units': 'V'},
'ADC_4B_3': {'desc': 'Channel 3 of ADC 0x4B', 'id': 151, 'units': 'V'},
'BATT': {'desc': 'Device battery status', 'id': '10', 'units': '%'},
'CCS811_ECO2': {'desc': 'Equivalent CO2', 'id': '112', 'units': 'ppm'},
'CCS811_VOCS': {'desc': 'total Volatile Organics Compounds',
'id': '113',
'units': 'ppm'},
'EXT_PM_A_1': {'desc': 'PM1 measurement from PMS5003 A',
'id': '71',
'units': 'ug/m3'},
'EXT_PM_A_10': {'desc': 'PM10 measurement from PMS5003 A',
'id': '73',
'units': 'ug/m3'},
'EXT_PM_A_25': {'desc': 'PM2.5 measurement from PMS5003 A',
'id': '72',
'units': 'ug/m3'},
'EXT_PM_B_1': {'desc': 'PM1 measurement from PMS5003 B',
'id': '75',
'units': 'ug/m3'},
'EXT_PM_B_10': {'desc': 'PM10 measurement from PMS5003 B',
'id': '77',
'units': 'ug/m3'},
'EXT_PM_B_25': {'desc': 'PM2.5 measurement from PMS5003 B',
'id': '76',
'units': 'ug/m3'},
'PM_1': {'desc': 'PM1 measurement from PMS5003',
'id': '89',
'units': 'ug/m3'},
'PM_10': {'desc': 'PM10 measurement from PMS5003',
'id': '88',
'units': 'ug/m3'},
'PM_25': {'desc': 'PM2.5 measurement from PMS5003',
'id': '87',
'units': 'ug/m3'},
'EXT_SHT31_HUM': {'desc': 'External SHT31 humidity',
'id': 'null',
'units': '%rh'},
'EXT_SHT31_TEMP': {'desc': 'External SHT31 temperature',
'id': 'null',
'units': 'degC'},
'PM_DALLAS_TEMP': {'desc': 'Dallas External probe Temperature',
'id': '96',
'units': 'degC'},
'GPS_LAT': {'desc': 'GPS lat', 'id': None, 'units': 'degC'},
'GPS_LONG': {'desc': 'GPS long', 'id': None, 'units': 'degC'},
'HUM': {'desc': 'Urban board humidity (SHT31)', 'id': '56', 'units': '%rh'},
'LIGHT': {'desc': 'Urban board ambient light', 'id': '14', 'units': 'lux'},
'NOISE_A': {'desc': 'A-scale noise SPL', 'id': '53', 'units': 'dBA'},
'PRESS': {'desc': 'Atmospheric pressure', 'id': '58', 'units': 'kPa'},
'TEMP': {'desc': 'Urban board temperature (SHT31)',
'id': '55',
'units': 'degC'},
'SCD30_TEMP': {'desc': 'External temperature (SCD30)',
'id': '160',
'units': 'degC'},
'SCD30_HUM': {'desc': 'External humidity (SCD30)',
'id': '161',
'units': '%rh'},
'SCD30_CO2': {'desc': 'CO2 (SCD30)', 'id': '158', 'units': 'ppm'}},
'source': None,
'sources': {'api': {'handler': 'ScApiDevice'},
'csv': {'header_skip': [1, 2, 3],
'index': 'TIME',
'sep': ',',
'tz-aware': True}},
'version': None},
'sck': {'clean_na': None,
'frequency': None,
'id': None,
'info_data_file': None,
'timezone': None,
'max_date': None,
'min_date': None,
'processed_data_file': None,
'raw_data_file': None,
'sensors': {'BATT': {'desc': 'Device battery status',
'id': '10',
'units': '%'},
'HUM': {'desc': 'Urban board humidity (SHT31)', 'id': '56', 'units': '%rh'},
'LIGHT': {'desc': 'Urban board ambient light', 'id': '14', 'units': 'lux'},
'TEMP': {'desc': 'Urban board temperature (SHT31)',
'id': '55',
'units': 'degC'}},
'source': None,
'sources': {'api': {'handler': 'ScApiDevice'},
'csv': {'header_skip': [1, 2, 3],
'index': 'TIME',
'sep': ',',
'tz-aware': False}},
'version': None},
'sck_15': {'clean_na': None,
'documentation': 'https://docs.smartcitizen.me/',
'frequency': None,
'id': None,
'timezone': None,
'max_date': None,
'min_date': None,
'sensors': {'BATT': {'desc': 'Device battery status',
'id': '10',
'units': '%'},
'HUM': {'desc': 'Urban board humidity (SHT31)', 'id': '56', 'units': '%rh'},
'LIGHT': {'desc': 'Urban board ambient light', 'id': '14', 'units': 'lux'},
'TEMP': {'desc': 'Urban board temperature (SHT31)',
'id': '55',
'units': 'degC'}},
'source': None,
'sources': {'api': {'handler': 'ScApiDevice'},
'csv': {'header_skip': None,
'index': 'Time',
'sep': ',',
'tz-aware': True}},
'version': None},
'sck_20': {'clean_na': None,
'documentation': 'https://docs.smartcitizen.me/',
'frequency': None,
'id': None,
'info_data_file': None,
'timezone': None,
'max_date': None,
'metrics': {'PM_10_CLEAN': {'desc': 'PM10 calculated based on both PMS5003 PM10 inputs',
'kwargs': {'limits': [0, 1000],
'name': 'PM_10',
'window_size': 5,
'window_type': None},
'process': 'clean_ts',
'units': 'ug/m3',
'post': False,
'id': None},
'PM_1_CLEAN': {'desc': 'PM1 calculated based on both PMS5003 PM10 inputs',
'kwargs': {'limits': [0, 1000],
'name': 'PM_1',
'window_size': 5,
'window_type': None},
'process': 'clean_ts',
'units': 'ug/m3',
'post': False,
'id': None},
'PM_25_CLEAN': {'desc': 'PM2.5 calculated based on both PMS5003 PM2.5 inputs',
'kwargs': {'limits': [0, 1000],
'name': 'PM_25',
'window_size': 5,
'window_type': None},
'process': 'clean_ts',
'units': 'ug/m3',
'post': False,
'id': None}},
'min_date': None,
'platform_id': 11,
'processed_data_file': None,
'raw_data_file': None,
'sensors': {'BATT': {'desc': 'Device battery status',
'id': '10',
'units': '%'},
'HUM': {'desc': 'Urban board humidity (SHT31)', 'id': '56', 'units': '%rh'},
'LIGHT': {'desc': 'Urban board ambient light', 'id': '14', 'units': 'lux'},
'NOISE_A': {'desc': 'A-scale noise SPL', 'id': '53', 'units': 'dBA'},
'PM_1': {'desc': 'PM1 measurement from PMS5003',
'id': '89',
'units': 'ug/m3'},
'PM_10': {'desc': 'PM10 measurement from PMS5003',
'id': '88',
'units': 'ug/m3'},
'PM_25': {'desc': 'PM2.5 measurement from PMS5003',
'id': '87',
'units': 'ug/m3'},
'PRESS': {'desc': 'Atmospheric pressure', 'id': '58', 'units': 'kPa'},
'TEMP': {'desc': 'Urban board temperature (SHT31)',
'id': '55',
'units': 'degC'}},
'source': None,
'sources': {'api': {'handler': 'ScApiDevice'},
'csv': {'header_skip': [1, 2, 3],
'index': 'TIME',
'sep': ',',
'tz-aware': True}},
'version': None},
'sck_21': {'clean_na': None,
'documentation': 'https://docs.smartcitizen.me/',
'frequency': None,
'id': None,
'info_data_file': None,
'timezone': None,
'max_date': None,
'metrics': {'CCS811_ECO2_CLEAN': {'desc': 'eCO2 cleaned data',
'kwargs': {'limits': [400, 65000],
'name': 'CCS811_ECO2',
'window_size': 5,
'window_type': None},
'process': 'clean_ts',
'units': 'ppm',
'post': False,
'id': None},
'CCS811_VOCS_CLEAN': {'desc': 'Volatile Organic Compounds cleaned data',
'kwargs': {'limits': [0, 65000],
'name': 'CCS811_VOCS',
'window_size': 5,
'window_type': None},
'process': 'clean_ts',
'units': 'ppb',
'post': False,
'id': None},
'PM_10_CLEAN': {'desc': 'PM10 calculated based on both PMS5003 PM10 inputs',
'kwargs': {'limits': [0, 1000],
'name': 'PM_10',
'window_size': 5,
'window_type': None},
'process': 'clean_ts',
'units': 'ug/m3',
'post': False,
'id': None},
'PM_1_CLEAN': {'desc': 'PM1 calculated based on both PMS5003 PM10 inputs',
'kwargs': {'limits': [0, 1000],
'name': 'PM_1',
'window_size': 5,
'window_type': None},
'process': 'clean_ts',
'units': 'ug/m3',
'post': False,
'id': None},
'PM_25_CLEAN': {'desc': 'PM2.5 calculated based on both PMS5003 PM2.5 inputs',
'kwargs': {'limits': [0, 1000],
'name': 'PM_25',
'window_size': 5,
'window_type': None},
'process': 'clean_ts',
'units': 'ug/m3',
'post': False,
'id': None}},
'min_date': None,
'platform_id': 28,
'processed_data_file': None,
'raw_data_file': None,
'sensors': {'BATT': {'desc': 'Device battery status',
'id': '10',
'units': '%'},
'CCS811_ECO2': {'desc': 'Equivalent CO2', 'id': '112', 'units': 'ppm'},
'CCS811_VOCS': {'desc': 'total Volatile Organics Compounds',
'id': '113',
'units': 'ppb'},
'HUM': {'desc': 'Urban board humidity (SHT31)', 'id': '56', 'units': '%rh'},
'LIGHT': {'desc': 'Urban board ambient light', 'id': '14', 'units': 'lux'},
'NOISE_A': {'desc': 'A-scale noise SPL', 'id': '53', 'units': 'dBA'},
'PM_1': {'desc': 'PM1 measurement from PMS5003',
'id': '89',
'units': 'ug/m3'},
'PM_10': {'desc': 'PM10 measurement from PMS5003',
'id': '88',
'units': 'ug/m3'},
'PM_25': {'desc': 'PM2.5 measurement from PMS5003',
'id': '87',
'units': 'ug/m3'},
'PRESS': {'desc': 'Atmospheric pressure', 'id': '58', 'units': 'kPa'},
'TEMP': {'desc': 'Urban board temperature (SHT31)',
'id': '55',
'units': 'degC'}},
'source': None,
'sources': {'api': {'handler': 'ScApiDevice'},
'csv': {'header_skip': [1, 2, 3],
'index': 'TIME',
'sep': ',',
'tz-aware': True}},
'version': None},
'sck_21_gps': {'clean_na': None,
'documentation': 'https://docs.smartcitizen.me/',
'frequency': None,
'id': None,
'info_data_file': None,
'timezone': None,
'max_date': None,
'metrics': {'CCS811_ECO2_CLEAN': {'desc': 'eCO2 cleaned data',
'kwargs': {'limits': [400, 65000],
'name': 'CCS811_ECO2',
'window_size': 5,
'window_type': None},
'process': 'clean_ts',
'units': 'ppm',
'post': False,
'id': None},
'CCS811_VOCS_CLEAN': {'desc': 'Volatile Organic Compounds cleaned data',
'kwargs': {'limits': [0, 65000],
'name': 'CCS811_VOCS',
'window_size': 5,
'window_type': None},
'process': 'clean_ts',
'units': 'ppb',
'post': False,
'id': None},
'PM_10_CLEAN': {'desc': 'PM10 calculated based on both PMS5003 PM10 inputs',
'kwargs': {'limits': [0, 1000],
'name': 'PM_10',
'window_size': 5,
'window_type': None},
'process': 'clean_ts',
'units': 'ug/m3',
'post': False,
'id': None},
'PM_1_CLEAN': {'desc': 'PM1 calculated based on both PMS5003 PM10 inputs',
'kwargs': {'limits': [0, 1000],
'name': 'PM_1',
'window_size': 5,
'window_type': None},
'process': 'clean_ts',
'units': 'ug/m3',
'post': False,
'id': None},
'PM_25_CLEAN': {'desc': 'PM2.5 calculated based on both PMS5003 PM2.5 inputs',
'kwargs': {'limits': [0, 1000],
'name': 'PM_25',
'window_size': 5,
'window_type': None},
'process': 'clean_ts',
'units': 'ug/m3',
'post': False,
'id': None}},
'min_date': None,
'platform_id': 28,
'processed_data_file': None,
'raw_data_file': None,
'sensors': {'BATT': {'desc': 'Device battery status',
'id': '10',
'units': '%'},
'CCS811_ECO2': {'desc': 'Equivalent CO2', 'id': '112', 'units': 'ppm'},
'CCS811_VOCS': {'desc': 'total Volatile Organics Compounds',
'id': '113',
'units': 'ppm'},
'GPS_ALT': {'desc': 'GPS Altitude', 'id': '127', 'units': 'm'},
'GPS_DIL': {'desc': 'GPS Horizontal Dilution of Position',
'id': '131',
'units': '#'},
'GPS_FIX_QUALITY': {'desc': 'GPS Fix Quality', 'id': '128', 'units': '#'},
'GPS_LAT': {'desc': 'GPS Latitude', 'id': '125', 'units': 'deg'},
'GPS_LONG': {'desc': 'GPS Longitude', 'id': '126', 'units': 'deg'},
'GPS_SAT_N': {'desc': 'GPS Traked Satellites', 'id': '130', 'units': '#'},
'GPS_SPEED': {'desc': 'GPS Speed', 'id': '129', 'units': 'm/s'},
'HUM': {'desc': 'Urban board humidity (SHT31)', 'id': '56', 'units': '%rh'},
'LIGHT': {'desc': 'Urban board ambient light', 'id': '14', 'units': 'lux'},
'NOISE_A': {'desc': 'A-scale noise SPL', 'id': '53', 'units': 'dBA'},
'PM_1': {'desc': 'PM1 measurement from PMS5003',
'id': '89',
'units': 'ug/m3'},
'PM_10': {'desc': 'PM10 measurement from PMS5003',
'id': '88',
'units': 'ug/m3'},
'PM_25': {'desc': 'PM2.5 measurement from PMS5003',
'id': '87',
'units': 'ug/m3'},
'PRESS': {'desc': 'Atmospheric pressure', 'id': '58', 'units': 'kPa'},
'TEMP': {'desc': 'Urban board temperature (SHT31)',
'id': '55',
'units': 'degC'}},
'source': None,
'sources': {'api': {'handler': 'ScApiDevice'},
'csv': {'header_skip': [1, 2, 3],
'index': 'TIME',
'sep': ',',
'tz-aware': True}},
'version': None},
'sck_21_nilu': {'clean_na': None,
'documentation': 'https://docs.smartcitizen.me/',
'frequency': None,
'id': None,
'timezone': None,
'max_date': None,
'metrics': None,
'min_date': None,
'processed_data_file': None,
'raw_data_file': None,
'sensors': {'BATT': {'desc': 'Device battery status',
'id': '60',
'units': '%'},
'CCS811_ECO2': {'desc': 'Equivalent CO2', 'id': '55', 'units': 'ppm'},
'CCS811_VOCS': {'desc': 'total Volatile Organics Compounds',
'id': '54',
'units': 'ppb'},
'HUM': {'desc': 'Urban board humidity (SHT31)', 'id': '5', 'units': '%rh'},
'LIGHT': {'desc': 'Urban board ambient light', 'id': '56', 'units': 'lux'},
'NOISE_A': {'desc': 'A-scale noise SPL', 'id': '57', 'units': 'dBA'},
'PM_1': {'desc': 'PM1 measurement from PMS5003',
'id': '11',
'units': 'ug/m3'},
'PM_10': {'desc': 'PM10 measurement from PMS5003',
'id': '12',
'units': 'ug/m3'},
'PM_25': {'desc': 'PM2.5 measurement from PMS5003',
'id': '13',
'units': 'ug/m3'},
'PRESS': {'desc': 'Atmospheric pressure', 'id': '1', 'units': 'kPa'},
'TEMP': {'desc': 'Urban board temperature (SHT31)',
'id': '15',
'units': 'degC'}},
'source': 'api',
'sources': {'api': {'handler': 'NiluApiDevice'},
'csv': {'header_skip': None,
'index': 'TIME',
'sep': ',',
'tz-aware': True}},
'version': None},
'sck_21_co2': {'clean_na': None,
'documentation': 'https://docs.smartcitizen.me/',
'frequency': None,
'id': None,
'info_data_file': None,
'timezone': None,
'max_date': None,
'metrics': {'CCS811_ECO2_CLEAN': {'desc': 'eCO2 cleaned data',
'kwargs': {'limits': [400, 65000],
'name': 'CCS811_ECO2',
'window_size': 5,
'window_type': None},
'process': 'clean_ts',
'units': 'ppm',
'post': False,
'id': None},
'CCS811_VOCS_CLEAN': {'desc': 'Volatile Organic Compounds cleaned data',
'kwargs': {'limits': [0, 65000],
'name': 'CCS811_VOCS',
'window_size': 5,
'window_type': None},
'process': 'clean_ts',
'units': 'ppb',
'post': False,
'id': None},
'PM_10_CLEAN': {'desc': 'PM10 calculated based on both PMS5003 PM10 inputs',
'kwargs': {'limits': [0, 1000],
'name': 'PM_10',
'window_size': 5,
'window_type': None},
'process': 'clean_ts',
'units': 'ug/m3',
'post': False,
'id': None},
'PM_1_CLEAN': {'desc': 'PM1 calculated based on both PMS5003 PM10 inputs',
'kwargs': {'limits': [0, 1000],
'name': 'PM_1',
'window_size': 5,
'window_type': None},
'process': 'clean_ts',
'units': 'ug/m3',
'post': False,
'id': None},
'PM_25_CLEAN': {'desc': 'PM2.5 calculated based on both PMS5003 PM2.5 inputs',
'kwargs': {'limits': [0, 1000],
'name': 'PM_25',
'window_size': 5,
'window_type': None},
'process': 'clean_ts',
'units': 'ug/m3',
'post': False,
'id': None}},
'min_date': None,
'platform_id': 28,
'processed_data_file': None,
'raw_data_file': None,
'sensors': {'BATT': {'desc': 'Device battery status',
'id': '10',
'units': '%'},
'CCS811_ECO2': {'desc': 'Equivalent CO2', 'id': '112', 'units': 'ppm'},
'CCS811_VOCS': {'desc': 'total Volatile Organics Compounds',
'id': '113',
'units': 'ppb'},
'HUM': {'desc': 'Urban board humidity (SHT31)', 'id': '56', 'units': '%rh'},
'LIGHT': {'desc': 'Urban board ambient light', 'id': '14', 'units': 'lux'},
'NOISE_A': {'desc': 'A-scale noise SPL', 'id': '53', 'units': 'dBA'},
'PM_1': {'desc': 'PM1 measurement from PMS5003',
'id': '89',
'units': 'ug/m3'},
'PM_10': {'desc': 'PM10 measurement from PMS5003',
'id': '88',
'units': 'ug/m3'},
'PM_25': {'desc': 'PM2.5 measurement from PMS5003',
'id': '87',
'units': 'ug/m3'},
'PRESS': {'desc': 'Atmospheric pressure', 'id': '58', 'units': 'kPa'},
'SCD30_TEMP': {'desc': 'External temperature (SCD30)',
'id': '160',
'units': 'degC'},
'SCD30_HUM': {'desc': 'External humidity (SCD30)',
'id': '161',
'units': '%rh'},
'SCD30_CO2': {'desc': 'CO2 (SCD30)', 'id': '158', 'units': 'ppm'},
'TEMP': {'desc': 'Urban board temperature (SHT31)',
'id': '55',
'units': 'degC'}},
'source': None,
'sources': {'api': {'handler': 'ScApiDevice'},
'csv': {'header_skip': [1, 2, 3],
'index': 'TIME',
'sep': ',',
'tz-aware': True}},
'version': None}}
config.blueprints.keys()
dict_keys(['base', 'csic_station', 'muv_station', 'sc_20_station_iscape', 'sc_21_station_iscape', 'sc_21_station_module', 'sck', 'sck_15', 'sck_20', 'sck_21', 'sck_21_gps', 'sck_21_nilu', 'sck_21_co2'])
This is how you can add a csv device (from sdcard data or other). The raw data file is to be put in ~/.cache/scdata/raw:
Load SD Card Data¶
test.add_device(Device(blueprint = 'sck_21' , descriptor = {'source': 'csv',
'id': 'csv_device',
'raw_data_file': 'example.csv',
'frequency': '1Min',
'timezone': 'Europe/Madrid'
}
)
)
[2022-05-30 12:25:47] - [SUCCESS] Device csv_device initialised
This instruction will create the test folder structure, the description and anything needed to keep track of the data:
# Create the test
test.create()
[2022-05-30 12:25:47] - [ERROR] Test already exists with this name. Full name: 2022_05_EXAMPLE_MINKE. Maybe force = True?
# Load it
test.load()
[2022-05-30 12:25:48] - [SUCCESS] Device 14638 initialised
[2022-05-30 12:25:48] - [SUCCESS] Hardware described in url is valid
[2022-05-30 12:25:48] - [WARNING] Skipping blueprint as it was defined in device constructor
[2022-05-30 12:25:48] - [SUCCESS] Device 14735 initialised
[2022-05-30 12:25:48] - [SUCCESS] Device csv_device initialised
[2022-05-30 12:25:49] - [WARNING] Removing sensors from device: ['ADC_48_0', 'ADC_48_1', 'ADC_48_2', 'ADC_48_3', 'ADC_49_0', 'ADC_49_1', 'ADC_49_2', 'ADC_49_3', 'ADC_4A_0', 'ADC_4A_1', 'ADC_4A_2', 'ADC_4A_3', 'ADC_4B_0', 'ADC_4B_1', 'ADC_4B_2', 'ADC_4B_3', 'EXT_PM_A_1', 'EXT_PM_A_10', 'EXT_PM_A_25', 'EXT_PM_B_1', 'EXT_PM_B_10', 'EXT_PM_B_25', 'PM_1', 'PM_10', 'PM_25', 'EXT_SHT31_HUM', 'EXT_SHT31_TEMP', 'PM_DALLAS_TEMP', 'GPS_LAT', 'GPS_LONG', 'SCD30_TEMP', 'SCD30_HUM', 'SCD30_CO2']
[2022-05-30 12:25:49] - [SUCCESS] Loaded cached files
[2022-05-30 12:25:49] - [SUCCESS] Admin Bearer found, using it
[2022-05-30 12:25:51] - [WARNING] No data in request for sensor: 89 (PM_1)
[2022-05-30 12:25:51] - [WARNING] No data in request for sensor: 88 (PM_10)
[2022-05-30 12:25:51] - [WARNING] No data in request for sensor: 87 (PM_25)
[2022-05-30 12:25:52] - [SUCCESS] Device 14638 loaded successfully from API
[2022-05-30 12:25:52] - [SUCCESS] Units check done
[2022-05-30 12:25:52] - [SUCCESS] File saved to:
/Users/macoscar/.cache/scdata/processed/2022/05/2022_05_EXAMPLE_MINKE/cached/14638.csv
[2022-05-30 12:25:52] - [SUCCESS] Device 14638 has been loaded
[2022-05-30 12:25:59] - [SUCCESS] Loaded cached files
[2022-05-30 12:25:59] - [SUCCESS] Admin Bearer found, using it
[2022-05-30 12:26:02] - [SUCCESS] Device 14735 loaded successfully from API
[2022-05-30 12:26:02] - [SUCCESS] Units check done
[2022-05-30 12:26:05] - [SUCCESS] File saved to:
/Users/macoscar/.cache/scdata/processed/2022/05/2022_05_EXAMPLE_MINKE/cached/14735.csv
[2022-05-30 12:26:05] - [SUCCESS] Device 14735 has been loaded
[2022-05-30 12:26:05] - [SUCCESS] Units check done
[2022-05-30 12:26:05] - [SUCCESS] Device csv_device has been loaded
[2022-05-30 12:26:05] - [SUCCESS] Test load done
# Check the devices
test.devices
{'14638': <scdata.device.Device at 0x10e7a5cd0>,
'14735': <scdata.device.Device at 0x13b129580>,
'csv_device': <scdata.device.Device at 0x10e7cd820>}
# Check the data
test.devices['14638'].readings
| BATT | CCS811_ECO2 | CCS811_VOCS | HUM | LIGHT | NOISE_A | PRESS | TEMP | |
|---|---|---|---|---|---|---|---|---|
| TIME | ||||||||
| 2021-10-19 11:28:00-05:00 | -1.0 | 400.0 | 0.0 | 61.33 | 143.0 | 87.02 | 84.93 | 24.08 |
| 2021-10-19 11:29:00-05:00 | -1.0 | 400.0 | 0.0 | 61.14 | 105.0 | 81.60 | 84.94 | 24.17 |
| 2021-10-19 11:30:00-05:00 | -1.0 | 400.0 | 0.0 | 60.34 | 89.0 | 81.91 | 84.93 | 24.31 |
| 2021-10-19 11:31:00-05:00 | -1.0 | 400.0 | 0.0 | 59.11 | 114.0 | 79.34 | 84.92 | 24.45 |
| 2021-10-19 11:32:00-05:00 | -1.0 | 400.0 | 0.0 | 59.20 | 113.0 | 85.20 | 84.93 | 24.58 |
| ... | ... | ... | ... | ... | ... | ... | ... | ... |
| 2021-10-27 06:26:00-05:00 | -1.0 | 1674.0 | 313.0 | 89.54 | 0.0 | 79.54 | 84.74 | 17.32 |
| 2021-10-27 06:27:00-05:00 | -1.0 | 1680.0 | 319.0 | 89.61 | 0.0 | 76.77 | 84.73 | 17.32 |
| 2021-10-27 06:28:00-05:00 | -1.0 | 1680.0 | 319.0 | 89.52 | 0.0 | 81.12 | 84.74 | 17.38 |
| 2021-10-27 06:29:00-05:00 | -1.0 | 1692.0 | 331.0 | 89.49 | 0.0 | 74.38 | 84.73 | 17.35 |
| 2021-10-27 06:30:00-05:00 | -1.0 | 1717.0 | 355.0 | 89.63 | 0.0 | 75.85 | 84.74 | 17.33 |
11223 rows × 8 columns
Plotting data¶
There are many options for plotting data. Here you can find some basic timeseries examples:
traces = {
"1": {"devices": "all",
"channel": "NOISE_A",
"subplot": 1},
}
figure = test.ts_plot(traces = traces,
options = {'frequency': '10Min', 'min_date': '2022-01'})
<Figure size 432x288 with 0 Axes>
traces = {
"1": {"devices": 'all',
"channel": "TEMP",
"subplot": 1},
"2": {"devices": 'all',
"channel": "NOISE_A",
"subplot": 2},
}
height = 300
f = test.ts_uplot(traces = traces,
formatting = {'width': height*2.5,
'height': height,
'padding-bottom': height,
'title': 'Temperature',
'ylabel': {1: 'Temperature (degC)',
2: 'Noise (dBA)'}
},
options = {'frequency': '10Min', 'html': False});
f
See also
For more examples, visit: https://github.com/fablabbcn/smartcitizen-data/blob/master/examples/notebooks/03_data_visualisation.ipynb
Understanding blueprints¶
Blueprints are ways of defining devices that include all the metadata necessary for that device, regarding units, processes needed to clean the data, file locations, even the documentation of the sensors. Blueprints data come from urls and are loaded while initialising the scdata. In the ~/.config/scdata/config.yaml file more urls can be defined and can be accessed by:
config.blueprints
{'base': {'clean_na': None,
'documentation': None,
'post_info': None,
'frequency': None,
'id': None,
'timezone': None,
'max_date': None,
'min_date': None,
'source': None,
'version': None},
'csic_station': {'clean_na': None,
'documentation': 'https://analisi.transparenciacatalunya.cat/',
'frequency': None,
'id': None,
'timezone': None,
'max_date': None,
'min_date': None,
'processed_data_file': None,
'raw_data_file': None,
'sensors': {'C6H6': {'id': 'C6H6', 'units': 'ug/m3'},
'CO': {'id': 'CO', 'units': 'mg/m3'},
'H2S': {'id': 'H2S', 'units': 'ug/m3'},
'NO': {'id': 'NO', 'units': 'ug/m3'},
'NO2': {'id': 'NO2', 'units': 'ug/m3'},
'NOX': {'id': 'NOX', 'units': 'ug/m3'},
'O3': {'id': 'O3', 'units': 'ug/m3'},
'PM_10': {'id': 'PM10', 'units': 'ug/m3'},
'PM_1': {'id': 'PM1', 'units': 'ug/m3'},
'PM_25': {'id': 'PM2.5', 'units': 'ug/m3'},
'SO2': {'id': 'SO2', 'units': 'ug/m3'}},
'source': None,
'sources': {'api': {'handler': 'DadesObertesApiDevice'},
'csv': {'header_skip': [1, 4],
'index': 'date',
'sep': ',',
'tz-aware': True}},
'version': None},
'muv_station': {'clean_na': None,
'documentation': 'https://github.com/waagsociety/air_quality_sensor_kit/tree/master/MUV%20Kit',
'frequency': None,
'id': None,
'info_data_file': None,
'timezone': None,
'max_date': None,
'min_date': None,
'processed_data_file': None,
'raw_data_file': None,
'sensors': {'GB_2A': {'id': 'no2op2', 'units': 'mV'},
'GB_2W': {'id': 'no2op1', 'units': 'mV'},
'GB_3A': {'id': 'o3op2', 'units': 'mV'},
'GB_3W': {'id': 'o3op1', 'units': 'mV'},
'HUM': {'id': 'h', 'units': '%rh'},
'NOISE_A': {'id': 'dB', 'units': 'dBA'},
'PM_10': {'id': 'p10', 'units': 'ug/m3'},
'PM_25': {'id': 'p25', 'units': 'ug/m3'},
'TEMP': {'id': 't', 'units': 'degC'}},
'source': None,
'sources': {'api': {'handler': 'MuvApiDevice'},
'csv': {'header_skip': None, 'index': 'Time'},
'sep': ',',
'tz-aware': True},
'version': None},
'sc_20_station_iscape': {'clean_na': None,
'documentation': 'https://docs.smartcitizen.me/',
'frequency': None,
'id': None,
'info_data_file': None,
'timezone': None,
'max_date': None,
'metrics': {'CO': {'desc': 'CO calculation based on simple electrode substraction',
'kwargs': {'auxiliary': 'GB_1A',
'hardware': 'alphadelta',
'id': None,
'pollutant': 'CO',
'working': 'GB_1W'},
'process': 'basic_4electrode_alg',
'units': 'ppm',
'post': False,
'id': None},
'EXT_PM_10_CLEAN': {'desc': 'PM10 calculated based on both PMS5003 PM10 inputs',
'kwargs': {'factor': 0.3,
'limits': [0, 1000],
'names': ['EXT_PM_A_10', 'EXT_PM_B_10'],
'pick': 'min',
'window_size': 5,
'window_type': None},
'process': 'merge_ts',
'units': 'ug/m3',
'post': False,
'id': None},
'EXT_PM_1_CLEAN': {'desc': 'PM1 calculated based on both PMS5003 PM1 inputs',
'kwargs': {'factor': 0.3,
'limits': [0, 1000],
'names': ['EXT_PM_A_1', 'EXT_PM_B_1'],
'pick': 'min',
'window_size': 5,
'window_type': None},
'process': 'merge_ts',
'units': 'ug/m3',
'post': False,
'id': None},
'EXT_PM_25_CLEAN': {'desc': 'PM2.5 calculated based on both PMS5003 PM2.5 inputs',
'kwargs': {'factor': 0.3,
'limits': [0, 1000],
'names': ['EXT_PM_A_25', 'EXT_PM_B_25'],
'pick': 'min',
'window_size': 5,
'window_type': None},
'process': 'merge_ts',
'units': 'ug/m3',
'post': False,
'id': None}},
'min_date': None,
'plaftorm_id': 19,
'processed_data_file': None,
'raw_data_file': None,
'sensors': {'BATT': {'desc': 'Device battery status',
'id': '10',
'units': '%'},
'EXT_PM_A_1': {'desc': 'PM1 measurement from PMS5003 A',
'id': '71',
'units': 'ug/m3'},
'EXT_PM_A_10': {'desc': 'PM10 measurement from PMS5003 A',
'id': '73',
'units': 'ug/m3'},
'EXT_PM_A_25': {'desc': 'PM2.5 measurement from PMS5003 A',
'id': '72',
'units': 'ug/m3'},
'EXT_PM_B_1': {'desc': 'PM1 measurement from PMS5003 B',
'id': '75',
'units': 'ug/m3'},
'EXT_PM_B_10': {'desc': 'PM10 measurement from PMS5003 B',
'id': '77',
'units': 'ug/m3'},
'EXT_PM_B_25': {'desc': 'PM2.5 measurement from PMS5003 B',
'id': '76',
'units': 'ug/m3'},
'EXT_TEMP': {'desc': 'Dallas External probe Temperature',
'id': '96',
'units': 'degC'},
'GB_1A': {'desc': 'Auxiliary electrode 1', 'id': '65', 'units': 'mV'},
'GB_1W': {'desc': 'Working electrode 1', 'id': '64', 'units': 'mV'},
'GB_2A': {'desc': 'Auxiliary electrode 2', 'id': '62', 'units': 'mV'},
'GB_2W': {'desc': 'Working electrode 2', 'id': '61', 'units': 'mV'},
'GB_3A': {'desc': 'Auxiliary electrode 3', 'id': '68', 'units': 'mV'},
'GB_3W': {'desc': 'Working electrode 3', 'id': '67', 'units': 'mV'},
'GB_HUM': {'desc': 'Gases board humidity (SHT31)',
'id': '80',
'units': '%rh'},
'GB_TEMP': {'desc': 'Gases board temperature (SHT31)',
'id': '79',
'units': 'degC'},
'HUM': {'desc': 'Urban board humidity (SHT31)', 'id': '56', 'units': '%rh'},
'LIGHT': {'desc': 'Urban board ambient light', 'id': '14', 'units': 'lux'},
'TEMP': {'desc': 'Urban board temperature (SHT31)',
'id': '55',
'units': 'degC'}},
'source': None,
'sources': {'api': {'handler': 'ScApiDevice'},
'csv': {'header_skip': [1, 2, 3],
'index': 'TIME',
'sep': ',',
'tz-aware': True}},
'version': None},
'sc_21_station_iscape': {'clean_na': None,
'documentation': 'https://docs.smartcitizen.me/',
'frequency': None,
'id': None,
'info_data_file': None,
'timezone': None,
'max_date': None,
'metrics': {'CCS811_ECO2_CLEAN': {'desc': 'eCO2 cleaned data',
'kwargs': {'limits': [400, 65000],
'name': 'CCS811_ECO2',
'window_size': 5,
'window_type': None},
'process': 'clean_ts',
'units': 'ppm',
'post': False,
'id': None},
'CCS811_VOCS_CLEAN': {'desc': 'Volatile Organic Compounds cleaned data',
'kwargs': {'limits': [0, 65000],
'name': 'CCS811_VOCS',
'window_size': 5,
'window_type': None},
'process': 'clean_ts',
'units': 'ppb',
'post': False,
'id': None},
'CO': {'desc': 'CO calculation based on simple electrode substraction',
'kwargs': {'auxiliary': 'GB_1A',
'hardware': 'alphadelta',
'id': None,
'pollutant': 'CO',
'working': 'GB_1W'},
'process': 'basic_4electrode_alg',
'units': 'ppm',
'post': False,
'id': None},
'EXT_PM_10_CLEAN': {'desc': 'PM10 calculated based on both PMS5003 PM10 inputs',
'kwargs': {'factor': 0.3,
'limits': [0, 1000],
'names': ['EXT_PM_A_10', 'EXT_PM_B_10'],
'pick': 'min',
'window_size': 5,
'window_type': None},
'process': 'merge_ts',
'units': 'ug/m3',
'post': False,
'id': None},
'EXT_PM_1_CLEAN': {'desc': 'PM1 calculated based on both PMS5003 PM1 inputs',
'kwargs': {'factor': 0.3,
'limits': [0, 1000],
'names': ['EXT_PM_A_1', 'EXT_PM_B_1'],
'pick': 'min',
'window_size': 5,
'window_type': None},
'process': 'merge_ts',
'units': 'ug/m3',
'post': False,
'id': None},
'EXT_PM_25_CLEAN': {'desc': 'PM2.5 calculated based on both PMS5003 PM2.5 inputs',
'kwargs': {'factor': 0.3,
'limits': [0, 1000],
'names': ['EXT_PM_A_25', 'EXT_PM_B_25'],
'pick': 'min',
'window_size': 5,
'window_type': None},
'process': 'merge_ts',
'units': 'ug/m3',
'post': False,
'id': None}},
'min_date': None,
'plaftorm_id': None,
'processed_data_file': None,
'raw_data_file': None,
'sensors': {'BATT': {'desc': 'Device battery status',
'id': '10',
'units': '%'},
'CCS811_ECO2': {'id': '112', 'units': 'ppm'},
'CCS811_VOCS': {'id': '113', 'units': 'ppm'},
'EXT_PM_A_1': {'desc': 'PM1 measurement from PMS5003 A',
'id': '71',
'units': 'ug/m3'},
'EXT_PM_A_10': {'desc': 'PM10 measurement from PMS5003 A',
'id': '73',
'units': 'ug/m3'},
'EXT_PM_A_25': {'desc': 'PM2.5 measurement from PMS5003 A',
'id': '72',
'units': 'ug/m3'},
'EXT_PM_B_1': {'desc': 'PM1 measurement from PMS5003 B',
'id': '75',
'units': 'ug/m3'},
'EXT_PM_B_10': {'desc': 'PM10 measurement from PMS5003 B',
'id': '77',
'units': 'ug/m3'},
'EXT_PM_B_25': {'desc': 'PM2.5 measurement from PMS5003 B',
'id': '76',
'units': 'ug/m3'},
'EXT_TEMP': {'desc': 'Dallas External probe Temperature',
'id': '96',
'units': 'degC'},
'GB_1A': {'desc': 'Auxiliary electrode 1', 'id': '65', 'units': 'mV'},
'GB_1W': {'desc': 'Working electrode 1', 'id': '64', 'units': 'mV'},
'GB_2A': {'desc': 'Auxiliary electrode 2', 'id': '62', 'units': 'mV'},
'GB_2W': {'desc': 'Working electrode 2', 'id': '61', 'units': 'mV'},
'GB_3A': {'desc': 'Auxiliary electrode 3', 'id': '68', 'units': 'mV'},
'GB_3W': {'desc': 'Working electrode 3', 'id': '67', 'units': 'mV'},
'GB_HUM': {'desc': 'Gases board humidity (SHT31)',
'id': '80',
'units': '%rh'},
'GB_TEMP': {'desc': 'Gases board temperature (SHT31)',
'id': '79',
'units': 'degC'},
'HUM': {'desc': 'Urban board humidity (SHT31)', 'id': '56', 'units': '%rh'},
'LIGHT': {'desc': 'Urban board ambient light', 'id': '14', 'units': 'lux'},
'NOISE_A': {'desc': 'A-scale noise SPL', 'id': '53', 'units': 'dBA'},
'TEMP': {'desc': 'Urban board temperature (SHT31)',
'id': '55',
'units': 'degC'}},
'source': None,
'sources': {'api': {'handler': 'ScApiDevice'},
'csv': {'header_skip': [1, 2, 3],
'index': 'TIME',
'sep': ',',
'tz-aware': True}},
'version': None},
'sc_21_station_module': {'clean_na': None,
'documentation': 'https://docs.smartcitizen.me/',
'frequency': None,
'id': None,
'info_data_file': None,
'timezone': None,
'max_date': None,
'metrics': {'CCS811_ECO2_CLEAN': {'desc': 'eCO2 cleaned data',
'kwargs': {'limits': [400, 65000],
'name': 'CCS811_ECO2',
'window_size': 5,
'window_type': None},
'process': 'clean_ts',
'units': 'ppm',
'post': False,
'id': None},
'CCS811_VOCS_CLEAN': {'desc': 'Volatile Organic Compounds cleaned data',
'kwargs': {'limits': [0, 65000],
'name': 'CCS811_VOCS',
'window_size': 5,
'window_type': None},
'process': 'clean_ts',
'units': 'ppb',
'post': False,
'id': None},
'EXT_PM_10_CLEAN': {'desc': 'PM10 calculated based on both PMS5003 PM10 inputs',
'kwargs': {'factor': 0.3,
'limits': [0, 1000],
'names': ['EXT_PM_A_10', 'EXT_PM_B_10'],
'pick': 'min',
'window_size': 5,
'window_type': None},
'process': 'merge_ts',
'units': 'ug/m3',
'post': True,
'id': 88},
'EXT_PM_1_CLEAN': {'desc': 'PM1 calculated based on both PMS5003 PM1 inputs',
'kwargs': {'factor': 0.3,
'limits': [0, 1000],
'names': ['EXT_PM_A_1', 'EXT_PM_B_1'],
'pick': 'min',
'window_size': 5,
'window_type': None},
'process': 'merge_ts',
'units': 'ug/m3',
'post': True,
'id': 89},
'EXT_PM_25_CLEAN': {'desc': 'PM2.5 calculated based on both PMS5003 PM2.5 inputs',
'kwargs': {'factor': 0.3,
'limits': [0, 1000],
'names': ['EXT_PM_A_25', 'EXT_PM_B_25'],
'pick': 'min',
'window_size': 5,
'window_type': None},
'process': 'merge_ts',
'units': 'ug/m3',
'post': True,
'id': 87},
'PT1000_POS': {'desc': 'PT1000 raw value',
'id': None,
'kwargs': {'channel': None},
'post': False,
'process': 'channel_names',
'units': 'V'},
'ASPT1000': {'desc': 'PT1000 temperature calculation in AFE',
'id': None,
'kwargs': {'pt1000minus': None,
'from_date': None,
'timezone': None,
'to_date': None,
'pt1000plus': None,
'afe_id': None},
'post': False,
'process': 'alphasense_pt1000',
'units': 'degC'},
'EC_SENSOR_TEMP': {'desc': 'Electrochemical sensor temperature',
'id': 'X013',
'kwargs': {'priority': 'ASPT1000'},
'post': False,
'process': 'ec_sensor_temp',
'units': 'degC'},
'CO_WE': {'desc': 'CO working electrode raw value',
'id': 'X001',
'kwargs': {'channel': None},
'post': False,
'process': 'channel_names',
'units': 'V'},
'CO_AE': {'desc': 'CO auxiliary electrode raw value',
'id': 'X002',
'kwargs': {'channel': None},
'post': False,
'process': 'channel_names',
'units': 'V'},
'NO2_WE': {'desc': 'NO2 working electrode raw value',
'id': 'X003',
'kwargs': {'channel': None},
'post': False,
'process': 'channel_names',
'units': 'V'},
'NO2_AE': {'desc': 'NO2 auxiliary electrode raw value',
'id': 'X004',
'kwargs': {'channel': None},
'post': False,
'process': 'channel_names',
'units': 'V'},
'NO_WE': {'desc': 'NO working electrode raw value',
'id': 'X005',
'kwargs': {'channel': None},
'post': False,
'process': 'channel_names',
'units': 'V'},
'NO_AE': {'desc': 'NO auxiliary electrode raw value',
'id': 'X006',
'kwargs': {'channel': None},
'post': False,
'process': 'channel_names',
'units': 'V'},
'SO2_WE': {'desc': 'SO2 working electrode raw value',
'id': 'X007',
'kwargs': {'channel': None},
'post': False,
'process': 'channel_names',
'units': 'V'},
'SO2_AE': {'desc': 'SO2 auxiliary electrode raw value',
'id': 'X008',
'kwargs': {'channel': None},
'post': False,
'process': 'channel_names',
'units': 'V'},
'H2S_WE': {'desc': 'H2S working electrode raw value',
'id': 'X009',
'kwargs': {'channel': None},
'post': False,
'process': 'channel_names',
'units': 'V'},
'H2S_AE': {'desc': 'H2S auxiliary electrode raw value',
'id': 'X010',
'kwargs': {'channel': None},
'post': False,
'process': 'channel_names',
'units': 'V'},
'OX_WE': {'desc': 'OX working electrode raw value',
'id': 'X011',
'kwargs': {'channel': None},
'post': False,
'process': 'channel_names',
'units': 'V'},
'OX_AE': {'desc': 'OX auxiliary electrode raw value',
'id': 'X012',
'kwargs': {'channel': None},
'post': False,
'process': 'channel_names',
'units': 'V'},
'CO': {'desc': 'Calculation of CO based on AAN 803-04',
'id': 152,
'kwargs': {'ae': None,
'alphasense_id': None,
'from_date': None,
'timezone': None,
't': 'EC_SENSOR_TEMP',
'to_date': None,
'we': None},
'post': True,
'process': 'alphasense_803_04',
'units': 'ppb'},
'NO2': {'desc': 'Calculation of NO2 based on AAN 803-04',
'id': 153,
'kwargs': {'ae': None,
'alphasense_id': None,
'from_date': None,
'timezone': None,
't': 'EC_SENSOR_TEMP',
'to_date': None,
'we': None},
'post': True,
'process': 'alphasense_803_04',
'units': 'ppb'},
'O3': {'desc': 'Calculation of O3 based on AAN 803-04',
'id': 157,
'kwargs': {'ae': None,
'alphasense_id': None,
'from_date': None,
'timezone': None,
't': 'EC_SENSOR_TEMP',
'to_date': None,
'we': None},
'post': True,
'process': 'alphasense_803_04',
'units': 'ppb'},
'SO2': {'desc': 'Calculation of SO2 based on AAN 803-04',
'id': 155,
'kwargs': {'ae': None,
'alphasense_id': None,
'from_date': None,
'timezone': None,
't': 'EC_SENSOR_TEMP',
'to_date': None,
'we': None,
'use_alternative': True},
'post': True,
'process': 'alphasense_803_04',
'units': 'ppb'},
'NO': {'desc': 'Calculation of NO based on AAN 803-04',
'id': 154,
'kwargs': {'ae': None,
'alphasense_id': None,
'from_date': None,
'timezone': None,
't': 'EC_SENSOR_TEMP',
'to_date': None,
'we': None},
'post': True,
'process': 'alphasense_803_04',
'units': 'ppb'},
'H2S': {'desc': 'Calculation of H2S based on AAN 803-04',
'id': 156,
'kwargs': {'ae': None,
'alphasense_id': None,
'from_date': None,
'timezone': None,
't': 'EC_SENSOR_TEMP',
'to_date': None,
'we': None},
'post': True,
'process': 'alphasense_803_04',
'units': 'ppb'}},
'min_date': None,
'platform_id': 28,
'processed_data_file': None,
'raw_data_file': None,
'sensors': {'ADC_48_0': {'desc': 'Channel 0 of ADC 0x48',
'id': 133,
'units': 'V'},
'ADC_48_1': {'desc': 'Channel 1 of ADC 0x48', 'id': 134, 'units': 'V'},
'ADC_48_2': {'desc': 'Channel 2 of ADC 0x48', 'id': 135, 'units': 'V'},
'ADC_48_3': {'desc': 'Channel 3 of ADC 0x48', 'id': 136, 'units': 'V'},
'ADC_49_0': {'desc': 'Channel 0 of ADC 0x49', 'id': 138, 'units': 'V'},
'ADC_49_1': {'desc': 'Channel 1 of ADC 0x49', 'id': 139, 'units': 'V'},
'ADC_49_2': {'desc': 'Channel 2 of ADC 0x49', 'id': 140, 'units': 'V'},
'ADC_49_3': {'desc': 'Channel 3 of ADC 0x49', 'id': 141, 'units': 'V'},
'ADC_4A_0': {'desc': 'Channel 0 of ADC 0x4A', 'id': 143, 'units': 'V'},
'ADC_4A_1': {'desc': 'Channel 1 of ADC 0x4A', 'id': 144, 'units': 'V'},
'ADC_4A_2': {'desc': 'Channel 2 of ADC 0x4A', 'id': 145, 'units': 'V'},
'ADC_4A_3': {'desc': 'Channel 3 of ADC 0x4A', 'id': 146, 'units': 'V'},
'ADC_4B_0': {'desc': 'Channel 0 of ADC 0x4B', 'id': 148, 'units': 'V'},
'ADC_4B_1': {'desc': 'Channel 1 of ADC 0x4B', 'id': 149, 'units': 'V'},
'ADC_4B_2': {'desc': 'Channel 2 of ADC 0x4B', 'id': 150, 'units': 'V'},
'ADC_4B_3': {'desc': 'Channel 3 of ADC 0x4B', 'id': 151, 'units': 'V'},
'BATT': {'desc': 'Device battery status', 'id': '10', 'units': '%'},
'CCS811_ECO2': {'desc': 'Equivalent CO2', 'id': '112', 'units': 'ppm'},
'CCS811_VOCS': {'desc': 'total Volatile Organics Compounds',
'id': '113',
'units': 'ppm'},
'EXT_PM_A_1': {'desc': 'PM1 measurement from PMS5003 A',
'id': '71',
'units': 'ug/m3'},
'EXT_PM_A_10': {'desc': 'PM10 measurement from PMS5003 A',
'id': '73',
'units': 'ug/m3'},
'EXT_PM_A_25': {'desc': 'PM2.5 measurement from PMS5003 A',
'id': '72',
'units': 'ug/m3'},
'EXT_PM_B_1': {'desc': 'PM1 measurement from PMS5003 B',
'id': '75',
'units': 'ug/m3'},
'EXT_PM_B_10': {'desc': 'PM10 measurement from PMS5003 B',
'id': '77',
'units': 'ug/m3'},
'EXT_PM_B_25': {'desc': 'PM2.5 measurement from PMS5003 B',
'id': '76',
'units': 'ug/m3'},
'PM_1': {'desc': 'PM1 measurement from PMS5003',
'id': '89',
'units': 'ug/m3'},
'PM_10': {'desc': 'PM10 measurement from PMS5003',
'id': '88',
'units': 'ug/m3'},
'PM_25': {'desc': 'PM2.5 measurement from PMS5003',
'id': '87',
'units': 'ug/m3'},
'EXT_SHT31_HUM': {'desc': 'External SHT31 humidity',
'id': 'null',
'units': '%rh'},
'EXT_SHT31_TEMP': {'desc': 'External SHT31 temperature',
'id': 'null',
'units': 'degC'},
'PM_DALLAS_TEMP': {'desc': 'Dallas External probe Temperature',
'id': '96',
'units': 'degC'},
'GPS_LAT': {'desc': 'GPS lat', 'id': None, 'units': 'degC'},
'GPS_LONG': {'desc': 'GPS long', 'id': None, 'units': 'degC'},
'HUM': {'desc': 'Urban board humidity (SHT31)', 'id': '56', 'units': '%rh'},
'LIGHT': {'desc': 'Urban board ambient light', 'id': '14', 'units': 'lux'},
'NOISE_A': {'desc': 'A-scale noise SPL', 'id': '53', 'units': 'dBA'},
'PRESS': {'desc': 'Atmospheric pressure', 'id': '58', 'units': 'kPa'},
'TEMP': {'desc': 'Urban board temperature (SHT31)',
'id': '55',
'units': 'degC'},
'SCD30_TEMP': {'desc': 'External temperature (SCD30)',
'id': '160',
'units': 'degC'},
'SCD30_HUM': {'desc': 'External humidity (SCD30)',
'id': '161',
'units': '%rh'},
'SCD30_CO2': {'desc': 'CO2 (SCD30)', 'id': '158', 'units': 'ppm'}},
'source': None,
'sources': {'api': {'handler': 'ScApiDevice'},
'csv': {'header_skip': [1, 2, 3],
'index': 'TIME',
'sep': ',',
'tz-aware': True}},
'version': None},
'sck': {'clean_na': None,
'frequency': None,
'id': None,
'info_data_file': None,
'timezone': None,
'max_date': None,
'min_date': None,
'processed_data_file': None,
'raw_data_file': None,
'sensors': {'BATT': {'desc': 'Device battery status',
'id': '10',
'units': '%'},
'HUM': {'desc': 'Urban board humidity (SHT31)', 'id': '56', 'units': '%rh'},
'LIGHT': {'desc': 'Urban board ambient light', 'id': '14', 'units': 'lux'},
'TEMP': {'desc': 'Urban board temperature (SHT31)',
'id': '55',
'units': 'degC'}},
'source': None,
'sources': {'api': {'handler': 'ScApiDevice'},
'csv': {'header_skip': [1, 2, 3],
'index': 'TIME',
'sep': ',',
'tz-aware': False}},
'version': None},
'sck_15': {'clean_na': None,
'documentation': 'https://docs.smartcitizen.me/',
'frequency': None,
'id': None,
'timezone': None,
'max_date': None,
'min_date': None,
'sensors': {'BATT': {'desc': 'Device battery status',
'id': '10',
'units': '%'},
'HUM': {'desc': 'Urban board humidity (SHT31)', 'id': '56', 'units': '%rh'},
'LIGHT': {'desc': 'Urban board ambient light', 'id': '14', 'units': 'lux'},
'TEMP': {'desc': 'Urban board temperature (SHT31)',
'id': '55',
'units': 'degC'}},
'source': None,
'sources': {'api': {'handler': 'ScApiDevice'},
'csv': {'header_skip': None,
'index': 'Time',
'sep': ',',
'tz-aware': True}},
'version': None},
'sck_20': {'clean_na': None,
'documentation': 'https://docs.smartcitizen.me/',
'frequency': None,
'id': None,
'info_data_file': None,
'timezone': None,
'max_date': None,
'metrics': {'PM_10_CLEAN': {'desc': 'PM10 calculated based on both PMS5003 PM10 inputs',
'kwargs': {'limits': [0, 1000],
'name': 'PM_10',
'window_size': 5,
'window_type': None},
'process': 'clean_ts',
'units': 'ug/m3',
'post': False,
'id': None},
'PM_1_CLEAN': {'desc': 'PM1 calculated based on both PMS5003 PM10 inputs',
'kwargs': {'limits': [0, 1000],
'name': 'PM_1',
'window_size': 5,
'window_type': None},
'process': 'clean_ts',
'units': 'ug/m3',
'post': False,
'id': None},
'PM_25_CLEAN': {'desc': 'PM2.5 calculated based on both PMS5003 PM2.5 inputs',
'kwargs': {'limits': [0, 1000],
'name': 'PM_25',
'window_size': 5,
'window_type': None},
'process': 'clean_ts',
'units': 'ug/m3',
'post': False,
'id': None}},
'min_date': None,
'platform_id': 11,
'processed_data_file': None,
'raw_data_file': None,
'sensors': {'BATT': {'desc': 'Device battery status',
'id': '10',
'units': '%'},
'HUM': {'desc': 'Urban board humidity (SHT31)', 'id': '56', 'units': '%rh'},
'LIGHT': {'desc': 'Urban board ambient light', 'id': '14', 'units': 'lux'},
'NOISE_A': {'desc': 'A-scale noise SPL', 'id': '53', 'units': 'dBA'},
'PM_1': {'desc': 'PM1 measurement from PMS5003',
'id': '89',
'units': 'ug/m3'},
'PM_10': {'desc': 'PM10 measurement from PMS5003',
'id': '88',
'units': 'ug/m3'},
'PM_25': {'desc': 'PM2.5 measurement from PMS5003',
'id': '87',
'units': 'ug/m3'},
'PRESS': {'desc': 'Atmospheric pressure', 'id': '58', 'units': 'kPa'},
'TEMP': {'desc': 'Urban board temperature (SHT31)',
'id': '55',
'units': 'degC'}},
'source': None,
'sources': {'api': {'handler': 'ScApiDevice'},
'csv': {'header_skip': [1, 2, 3],
'index': 'TIME',
'sep': ',',
'tz-aware': True}},
'version': None},
'sck_21': {'clean_na': None,
'documentation': 'https://docs.smartcitizen.me/',
'frequency': None,
'id': None,
'info_data_file': None,
'timezone': None,
'max_date': None,
'metrics': {'CCS811_ECO2_CLEAN': {'desc': 'eCO2 cleaned data',
'kwargs': {'limits': [400, 65000],
'name': 'CCS811_ECO2',
'window_size': 5,
'window_type': None},
'process': 'clean_ts',
'units': 'ppm',
'post': False,
'id': None},
'CCS811_VOCS_CLEAN': {'desc': 'Volatile Organic Compounds cleaned data',
'kwargs': {'limits': [0, 65000],
'name': 'CCS811_VOCS',
'window_size': 5,
'window_type': None},
'process': 'clean_ts',
'units': 'ppb',
'post': False,
'id': None},
'PM_10_CLEAN': {'desc': 'PM10 calculated based on both PMS5003 PM10 inputs',
'kwargs': {'limits': [0, 1000],
'name': 'PM_10',
'window_size': 5,
'window_type': None},
'process': 'clean_ts',
'units': 'ug/m3',
'post': False,
'id': None},
'PM_1_CLEAN': {'desc': 'PM1 calculated based on both PMS5003 PM10 inputs',
'kwargs': {'limits': [0, 1000],
'name': 'PM_1',
'window_size': 5,
'window_type': None},
'process': 'clean_ts',
'units': 'ug/m3',
'post': False,
'id': None},
'PM_25_CLEAN': {'desc': 'PM2.5 calculated based on both PMS5003 PM2.5 inputs',
'kwargs': {'limits': [0, 1000],
'name': 'PM_25',
'window_size': 5,
'window_type': None},
'process': 'clean_ts',
'units': 'ug/m3',
'post': False,
'id': None}},
'min_date': None,
'platform_id': 28,
'processed_data_file': None,
'raw_data_file': None,
'sensors': {'BATT': {'desc': 'Device battery status',
'id': '10',
'units': '%'},
'CCS811_ECO2': {'desc': 'Equivalent CO2', 'id': '112', 'units': 'ppm'},
'CCS811_VOCS': {'desc': 'total Volatile Organics Compounds',
'id': '113',
'units': 'ppb'},
'HUM': {'desc': 'Urban board humidity (SHT31)', 'id': '56', 'units': '%rh'},
'LIGHT': {'desc': 'Urban board ambient light', 'id': '14', 'units': 'lux'},
'NOISE_A': {'desc': 'A-scale noise SPL', 'id': '53', 'units': 'dBA'},
'PM_1': {'desc': 'PM1 measurement from PMS5003',
'id': '89',
'units': 'ug/m3'},
'PM_10': {'desc': 'PM10 measurement from PMS5003',
'id': '88',
'units': 'ug/m3'},
'PM_25': {'desc': 'PM2.5 measurement from PMS5003',
'id': '87',
'units': 'ug/m3'},
'PRESS': {'desc': 'Atmospheric pressure', 'id': '58', 'units': 'kPa'},
'TEMP': {'desc': 'Urban board temperature (SHT31)',
'id': '55',
'units': 'degC'}},
'source': None,
'sources': {'api': {'handler': 'ScApiDevice'},
'csv': {'header_skip': [1, 2, 3],
'index': 'TIME',
'sep': ',',
'tz-aware': True}},
'version': None},
'sck_21_gps': {'clean_na': None,
'documentation': 'https://docs.smartcitizen.me/',
'frequency': None,
'id': None,
'info_data_file': None,
'timezone': None,
'max_date': None,
'metrics': {'CCS811_ECO2_CLEAN': {'desc': 'eCO2 cleaned data',
'kwargs': {'limits': [400, 65000],
'name': 'CCS811_ECO2',
'window_size': 5,
'window_type': None},
'process': 'clean_ts',
'units': 'ppm',
'post': False,
'id': None},
'CCS811_VOCS_CLEAN': {'desc': 'Volatile Organic Compounds cleaned data',
'kwargs': {'limits': [0, 65000],
'name': 'CCS811_VOCS',
'window_size': 5,
'window_type': None},
'process': 'clean_ts',
'units': 'ppb',
'post': False,
'id': None},
'PM_10_CLEAN': {'desc': 'PM10 calculated based on both PMS5003 PM10 inputs',
'kwargs': {'limits': [0, 1000],
'name': 'PM_10',
'window_size': 5,
'window_type': None},
'process': 'clean_ts',
'units': 'ug/m3',
'post': False,
'id': None},
'PM_1_CLEAN': {'desc': 'PM1 calculated based on both PMS5003 PM10 inputs',
'kwargs': {'limits': [0, 1000],
'name': 'PM_1',
'window_size': 5,
'window_type': None},
'process': 'clean_ts',
'units': 'ug/m3',
'post': False,
'id': None},
'PM_25_CLEAN': {'desc': 'PM2.5 calculated based on both PMS5003 PM2.5 inputs',
'kwargs': {'limits': [0, 1000],
'name': 'PM_25',
'window_size': 5,
'window_type': None},
'process': 'clean_ts',
'units': 'ug/m3',
'post': False,
'id': None}},
'min_date': None,
'platform_id': 28,
'processed_data_file': None,
'raw_data_file': None,
'sensors': {'BATT': {'desc': 'Device battery status',
'id': '10',
'units': '%'},
'CCS811_ECO2': {'desc': 'Equivalent CO2', 'id': '112', 'units': 'ppm'},
'CCS811_VOCS': {'desc': 'total Volatile Organics Compounds',
'id': '113',
'units': 'ppm'},
'GPS_ALT': {'desc': 'GPS Altitude', 'id': '127', 'units': 'm'},
'GPS_DIL': {'desc': 'GPS Horizontal Dilution of Position',
'id': '131',
'units': '#'},
'GPS_FIX_QUALITY': {'desc': 'GPS Fix Quality', 'id': '128', 'units': '#'},
'GPS_LAT': {'desc': 'GPS Latitude', 'id': '125', 'units': 'deg'},
'GPS_LONG': {'desc': 'GPS Longitude', 'id': '126', 'units': 'deg'},
'GPS_SAT_N': {'desc': 'GPS Traked Satellites', 'id': '130', 'units': '#'},
'GPS_SPEED': {'desc': 'GPS Speed', 'id': '129', 'units': 'm/s'},
'HUM': {'desc': 'Urban board humidity (SHT31)', 'id': '56', 'units': '%rh'},
'LIGHT': {'desc': 'Urban board ambient light', 'id': '14', 'units': 'lux'},
'NOISE_A': {'desc': 'A-scale noise SPL', 'id': '53', 'units': 'dBA'},
'PM_1': {'desc': 'PM1 measurement from PMS5003',
'id': '89',
'units': 'ug/m3'},
'PM_10': {'desc': 'PM10 measurement from PMS5003',
'id': '88',
'units': 'ug/m3'},
'PM_25': {'desc': 'PM2.5 measurement from PMS5003',
'id': '87',
'units': 'ug/m3'},
'PRESS': {'desc': 'Atmospheric pressure', 'id': '58', 'units': 'kPa'},
'TEMP': {'desc': 'Urban board temperature (SHT31)',
'id': '55',
'units': 'degC'}},
'source': None,
'sources': {'api': {'handler': 'ScApiDevice'},
'csv': {'header_skip': [1, 2, 3],
'index': 'TIME',
'sep': ',',
'tz-aware': True}},
'version': None},
'sck_21_nilu': {'clean_na': None,
'documentation': 'https://docs.smartcitizen.me/',
'frequency': None,
'id': None,
'timezone': None,
'max_date': None,
'metrics': None,
'min_date': None,
'processed_data_file': None,
'raw_data_file': None,
'sensors': {'BATT': {'desc': 'Device battery status',
'id': '60',
'units': '%'},
'CCS811_ECO2': {'desc': 'Equivalent CO2', 'id': '55', 'units': 'ppm'},
'CCS811_VOCS': {'desc': 'total Volatile Organics Compounds',
'id': '54',
'units': 'ppb'},
'HUM': {'desc': 'Urban board humidity (SHT31)', 'id': '5', 'units': '%rh'},
'LIGHT': {'desc': 'Urban board ambient light', 'id': '56', 'units': 'lux'},
'NOISE_A': {'desc': 'A-scale noise SPL', 'id': '57', 'units': 'dBA'},
'PM_1': {'desc': 'PM1 measurement from PMS5003',
'id': '11',
'units': 'ug/m3'},
'PM_10': {'desc': 'PM10 measurement from PMS5003',
'id': '12',
'units': 'ug/m3'},
'PM_25': {'desc': 'PM2.5 measurement from PMS5003',
'id': '13',
'units': 'ug/m3'},
'PRESS': {'desc': 'Atmospheric pressure', 'id': '1', 'units': 'kPa'},
'TEMP': {'desc': 'Urban board temperature (SHT31)',
'id': '15',
'units': 'degC'}},
'source': 'api',
'sources': {'api': {'handler': 'NiluApiDevice'},
'csv': {'header_skip': None,
'index': 'TIME',
'sep': ',',
'tz-aware': True}},
'version': None},
'sck_21_co2': {'clean_na': None,
'documentation': 'https://docs.smartcitizen.me/',
'frequency': None,
'id': None,
'info_data_file': None,
'timezone': None,
'max_date': None,
'metrics': {'CCS811_ECO2_CLEAN': {'desc': 'eCO2 cleaned data',
'kwargs': {'limits': [400, 65000],
'name': 'CCS811_ECO2',
'window_size': 5,
'window_type': None},
'process': 'clean_ts',
'units': 'ppm',
'post': False,
'id': None},
'CCS811_VOCS_CLEAN': {'desc': 'Volatile Organic Compounds cleaned data',
'kwargs': {'limits': [0, 65000],
'name': 'CCS811_VOCS',
'window_size': 5,
'window_type': None},
'process': 'clean_ts',
'units': 'ppb',
'post': False,
'id': None},
'PM_10_CLEAN': {'desc': 'PM10 calculated based on both PMS5003 PM10 inputs',
'kwargs': {'limits': [0, 1000],
'name': 'PM_10',
'window_size': 5,
'window_type': None},
'process': 'clean_ts',
'units': 'ug/m3',
'post': False,
'id': None},
'PM_1_CLEAN': {'desc': 'PM1 calculated based on both PMS5003 PM10 inputs',
'kwargs': {'limits': [0, 1000],
'name': 'PM_1',
'window_size': 5,
'window_type': None},
'process': 'clean_ts',
'units': 'ug/m3',
'post': False,
'id': None},
'PM_25_CLEAN': {'desc': 'PM2.5 calculated based on both PMS5003 PM2.5 inputs',
'kwargs': {'limits': [0, 1000],
'name': 'PM_25',
'window_size': 5,
'window_type': None},
'process': 'clean_ts',
'units': 'ug/m3',
'post': False,
'id': None}},
'min_date': None,
'platform_id': 28,
'processed_data_file': None,
'raw_data_file': None,
'sensors': {'BATT': {'desc': 'Device battery status',
'id': '10',
'units': '%'},
'CCS811_ECO2': {'desc': 'Equivalent CO2', 'id': '112', 'units': 'ppm'},
'CCS811_VOCS': {'desc': 'total Volatile Organics Compounds',
'id': '113',
'units': 'ppb'},
'HUM': {'desc': 'Urban board humidity (SHT31)', 'id': '56', 'units': '%rh'},
'LIGHT': {'desc': 'Urban board ambient light', 'id': '14', 'units': 'lux'},
'NOISE_A': {'desc': 'A-scale noise SPL', 'id': '53', 'units': 'dBA'},
'PM_1': {'desc': 'PM1 measurement from PMS5003',
'id': '89',
'units': 'ug/m3'},
'PM_10': {'desc': 'PM10 measurement from PMS5003',
'id': '88',
'units': 'ug/m3'},
'PM_25': {'desc': 'PM2.5 measurement from PMS5003',
'id': '87',
'units': 'ug/m3'},
'PRESS': {'desc': 'Atmospheric pressure', 'id': '58', 'units': 'kPa'},
'SCD30_TEMP': {'desc': 'External temperature (SCD30)',
'id': '160',
'units': 'degC'},
'SCD30_HUM': {'desc': 'External humidity (SCD30)',
'id': '161',
'units': '%rh'},
'SCD30_CO2': {'desc': 'CO2 (SCD30)', 'id': '158', 'units': 'ppm'},
'TEMP': {'desc': 'Urban board temperature (SHT31)',
'id': '55',
'units': 'degC'}},
'source': None,
'sources': {'api': {'handler': 'ScApiDevice'},
'csv': {'header_skip': [1, 2, 3],
'index': 'TIME',
'sep': ',',
'tz-aware': True}},
'version': None}}
config.blueprints['sck_21']
{'clean_na': None,
'documentation': 'https://docs.smartcitizen.me/',
'frequency': None,
'id': None,
'info_data_file': None,
'timezone': None,
'max_date': None,
'metrics': {'CCS811_ECO2_CLEAN': {'desc': 'eCO2 cleaned data',
'kwargs': {'limits': [400, 65000],
'name': 'CCS811_ECO2',
'window_size': 5,
'window_type': None},
'process': 'clean_ts',
'units': 'ppm',
'post': False,
'id': None},
'CCS811_VOCS_CLEAN': {'desc': 'Volatile Organic Compounds cleaned data',
'kwargs': {'limits': [0, 65000],
'name': 'CCS811_VOCS',
'window_size': 5,
'window_type': None},
'process': 'clean_ts',
'units': 'ppb',
'post': False,
'id': None},
'PM_10_CLEAN': {'desc': 'PM10 calculated based on both PMS5003 PM10 inputs',
'kwargs': {'limits': [0, 1000],
'name': 'PM_10',
'window_size': 5,
'window_type': None},
'process': 'clean_ts',
'units': 'ug/m3',
'post': False,
'id': None},
'PM_1_CLEAN': {'desc': 'PM1 calculated based on both PMS5003 PM10 inputs',
'kwargs': {'limits': [0, 1000],
'name': 'PM_1',
'window_size': 5,
'window_type': None},
'process': 'clean_ts',
'units': 'ug/m3',
'post': False,
'id': None},
'PM_25_CLEAN': {'desc': 'PM2.5 calculated based on both PMS5003 PM2.5 inputs',
'kwargs': {'limits': [0, 1000],
'name': 'PM_25',
'window_size': 5,
'window_type': None},
'process': 'clean_ts',
'units': 'ug/m3',
'post': False,
'id': None}},
'min_date': None,
'platform_id': 28,
'processed_data_file': None,
'raw_data_file': None,
'sensors': {'BATT': {'desc': 'Device battery status',
'id': '10',
'units': '%'},
'CCS811_ECO2': {'desc': 'Equivalent CO2', 'id': '112', 'units': 'ppm'},
'CCS811_VOCS': {'desc': 'total Volatile Organics Compounds',
'id': '113',
'units': 'ppb'},
'HUM': {'desc': 'Urban board humidity (SHT31)', 'id': '56', 'units': '%rh'},
'LIGHT': {'desc': 'Urban board ambient light', 'id': '14', 'units': 'lux'},
'NOISE_A': {'desc': 'A-scale noise SPL', 'id': '53', 'units': 'dBA'},
'PM_1': {'desc': 'PM1 measurement from PMS5003',
'id': '89',
'units': 'ug/m3'},
'PM_10': {'desc': 'PM10 measurement from PMS5003',
'id': '88',
'units': 'ug/m3'},
'PM_25': {'desc': 'PM2.5 measurement from PMS5003',
'id': '87',
'units': 'ug/m3'},
'PRESS': {'desc': 'Atmospheric pressure', 'id': '58', 'units': 'kPa'},
'TEMP': {'desc': 'Urban board temperature (SHT31)',
'id': '55',
'units': 'degC'}},
'source': None,
'sources': {'api': {'handler': 'ScApiDevice'},
'csv': {'header_skip': [1, 2, 3],
'index': 'TIME',
'sep': ',',
'tz-aware': True}},
'version': None}
Each of the properties in a device is defined in it’s blueprint. For instance, the sensors available.
config.blueprints['sck_21']['sensors']
{'BATT': {'desc': 'Device battery status', 'id': '10', 'units': '%'},
'CCS811_ECO2': {'desc': 'Equivalent CO2', 'id': '112', 'units': 'ppm'},
'CCS811_VOCS': {'desc': 'total Volatile Organics Compounds',
'id': '113',
'units': 'ppb'},
'HUM': {'desc': 'Urban board humidity (SHT31)', 'id': '56', 'units': '%rh'},
'LIGHT': {'desc': 'Urban board ambient light', 'id': '14', 'units': 'lux'},
'NOISE_A': {'desc': 'A-scale noise SPL', 'id': '53', 'units': 'dBA'},
'PM_1': {'desc': 'PM1 measurement from PMS5003',
'id': '89',
'units': 'ug/m3'},
'PM_10': {'desc': 'PM10 measurement from PMS5003',
'id': '88',
'units': 'ug/m3'},
'PM_25': {'desc': 'PM2.5 measurement from PMS5003',
'id': '87',
'units': 'ug/m3'},
'PRESS': {'desc': 'Atmospheric pressure', 'id': '58', 'units': 'kPa'},
'TEMP': {'desc': 'Urban board temperature (SHT31)',
'id': '55',
'units': 'degC'}}
In the case of devices coming from the SC platform, these sensors are defined based on these definitions, and are meant for traceability of the data, explaining what each device contains. For other devices, this has to be filled out manually.
The bluepring also contains the metrics (or processed channels) associated with the sensors. These metrics will be calculated once test.process() or device.process() are called
config.blueprints['sck_21']['metrics']
{'CCS811_ECO2_CLEAN': {'desc': 'eCO2 cleaned data',
'kwargs': {'limits': [400, 65000],
'name': 'CCS811_ECO2',
'window_size': 5,
'window_type': None},
'process': 'clean_ts',
'units': 'ppm',
'post': False,
'id': None},
'CCS811_VOCS_CLEAN': {'desc': 'Volatile Organic Compounds cleaned data',
'kwargs': {'limits': [0, 65000],
'name': 'CCS811_VOCS',
'window_size': 5,
'window_type': None},
'process': 'clean_ts',
'units': 'ppb',
'post': False,
'id': None},
'PM_10_CLEAN': {'desc': 'PM10 calculated based on both PMS5003 PM10 inputs',
'kwargs': {'limits': [0, 1000],
'name': 'PM_10',
'window_size': 5,
'window_type': None},
'process': 'clean_ts',
'units': 'ug/m3',
'post': False,
'id': None},
'PM_1_CLEAN': {'desc': 'PM1 calculated based on both PMS5003 PM10 inputs',
'kwargs': {'limits': [0, 1000],
'name': 'PM_1',
'window_size': 5,
'window_type': None},
'process': 'clean_ts',
'units': 'ug/m3',
'post': False,
'id': None},
'PM_25_CLEAN': {'desc': 'PM2.5 calculated based on both PMS5003 PM2.5 inputs',
'kwargs': {'limits': [0, 1000],
'name': 'PM_25',
'window_size': 5,
'window_type': None},
'process': 'clean_ts',
'units': 'ug/m3',
'post': False,
'id': None}}
Finally, to add your own blueprint, you can add the url directly in the config.yaml file or by:
config.blueprints_urls.append('another_url')
config.get_meta_data()
Failed request. Probably no connection or invalid json file
Data structure¶
Here we show how the data is structured. A test contains devices, metadata and models (if created, see sensor_calibration_workflows.ipynb)
First the tests:
list(test.devices.keys())
['14638', '14735', 'csv_device']
Then the data inside the devices is simply a pandas.DataFrame()
test.devices['14638'].readings.head(5)
| BATT | CCS811_ECO2 | CCS811_VOCS | HUM | LIGHT | NOISE_A | PRESS | TEMP | |
|---|---|---|---|---|---|---|---|---|
| TIME | ||||||||
| 2021-10-19 11:28:00-05:00 | -1.0 | 400.0 | 0.0 | 61.33 | 143.0 | 87.02 | 84.93 | 24.08 |
| 2021-10-19 11:29:00-05:00 | -1.0 | 400.0 | 0.0 | 61.14 | 105.0 | 81.60 | 84.94 | 24.17 |
| 2021-10-19 11:30:00-05:00 | -1.0 | 400.0 | 0.0 | 60.34 | 89.0 | 81.91 | 84.93 | 24.31 |
| 2021-10-19 11:31:00-05:00 | -1.0 | 400.0 | 0.0 | 59.11 | 114.0 | 79.34 | 84.92 | 24.45 |
| 2021-10-19 11:32:00-05:00 | -1.0 | 400.0 | 0.0 | 59.20 | 113.0 | 85.20 | 84.93 | 24.58 |
The test description information can be accessed and modified in the test.descriptor dict(). You can also access where the data is with test.path
test.descriptor
{'author': '',
'comment': '',
'commit': '',
'devices': {'14638': {'skip_blueprint': True,
'blueprint': 'sc_21_station_module',
'clean_na': None,
'documentation': 'https://docs.smartcitizen.me/',
'frequency': None,
'id': '14638',
'info_data_file': None,
'timezone': 'America/Mexico_City',
'max_date': '2021-10-30',
'metrics': {'CCS811_ECO2_CLEAN': {'desc': 'eCO2 cleaned data',
'kwargs': {'limits': [400, 65000],
'name': 'CCS811_ECO2',
'window_size': 5,
'window_type': None},
'process': 'clean_ts',
'units': 'ppm',
'post': False,
'id': None},
'CCS811_VOCS_CLEAN': {'desc': 'Volatile Organic Compounds cleaned data',
'kwargs': {'limits': [0, 65000],
'name': 'CCS811_VOCS',
'window_size': 5,
'window_type': None},
'process': 'clean_ts',
'units': 'ppb',
'post': False,
'id': None},
'EXT_PM_10_CLEAN': {'desc': 'PM10 calculated based on both PMS5003 PM10 inputs',
'kwargs': {'factor': 0.3,
'limits': [0, 1000],
'names': ['EXT_PM_A_10', 'EXT_PM_B_10'],
'pick': 'min',
'window_size': 5,
'window_type': None},
'process': 'merge_ts',
'units': 'ug/m3',
'post': True,
'id': 88},
'EXT_PM_1_CLEAN': {'desc': 'PM1 calculated based on both PMS5003 PM1 inputs',
'kwargs': {'factor': 0.3,
'limits': [0, 1000],
'names': ['EXT_PM_A_1', 'EXT_PM_B_1'],
'pick': 'min',
'window_size': 5,
'window_type': None},
'process': 'merge_ts',
'units': 'ug/m3',
'post': True,
'id': 89},
'EXT_PM_25_CLEAN': {'desc': 'PM2.5 calculated based on both PMS5003 PM2.5 inputs',
'kwargs': {'factor': 0.3,
'limits': [0, 1000],
'names': ['EXT_PM_A_25', 'EXT_PM_B_25'],
'pick': 'min',
'window_size': 5,
'window_type': None},
'process': 'merge_ts',
'units': 'ug/m3',
'post': True,
'id': 87},
'PT1000_POS': {'desc': 'PT1000 raw value',
'id': None,
'kwargs': {'channel': None},
'post': False,
'process': 'channel_names',
'units': 'V'},
'ASPT1000': {'desc': 'PT1000 temperature calculation in AFE',
'id': None,
'kwargs': {'pt1000minus': None,
'from_date': None,
'timezone': None,
'to_date': None,
'pt1000plus': None,
'afe_id': None},
'post': False,
'process': 'alphasense_pt1000',
'units': 'degC'},
'EC_SENSOR_TEMP': {'desc': 'Electrochemical sensor temperature',
'id': 'X013',
'kwargs': {'priority': 'ASPT1000'},
'post': False,
'process': 'ec_sensor_temp',
'units': 'degC'},
'CO_WE': {'desc': 'CO working electrode raw value',
'id': 'X001',
'kwargs': {'channel': None},
'post': False,
'process': 'channel_names',
'units': 'V'},
'CO_AE': {'desc': 'CO auxiliary electrode raw value',
'id': 'X002',
'kwargs': {'channel': None},
'post': False,
'process': 'channel_names',
'units': 'V'},
'NO2_WE': {'desc': 'NO2 working electrode raw value',
'id': 'X003',
'kwargs': {'channel': None},
'post': False,
'process': 'channel_names',
'units': 'V'},
'NO2_AE': {'desc': 'NO2 auxiliary electrode raw value',
'id': 'X004',
'kwargs': {'channel': None},
'post': False,
'process': 'channel_names',
'units': 'V'},
'NO_WE': {'desc': 'NO working electrode raw value',
'id': 'X005',
'kwargs': {'channel': None},
'post': False,
'process': 'channel_names',
'units': 'V'},
'NO_AE': {'desc': 'NO auxiliary electrode raw value',
'id': 'X006',
'kwargs': {'channel': None},
'post': False,
'process': 'channel_names',
'units': 'V'},
'SO2_WE': {'desc': 'SO2 working electrode raw value',
'id': 'X007',
'kwargs': {'channel': None},
'post': False,
'process': 'channel_names',
'units': 'V'},
'SO2_AE': {'desc': 'SO2 auxiliary electrode raw value',
'id': 'X008',
'kwargs': {'channel': None},
'post': False,
'process': 'channel_names',
'units': 'V'},
'H2S_WE': {'desc': 'H2S working electrode raw value',
'id': 'X009',
'kwargs': {'channel': None},
'post': False,
'process': 'channel_names',
'units': 'V'},
'H2S_AE': {'desc': 'H2S auxiliary electrode raw value',
'id': 'X010',
'kwargs': {'channel': None},
'post': False,
'process': 'channel_names',
'units': 'V'},
'OX_WE': {'desc': 'OX working electrode raw value',
'id': 'X011',
'kwargs': {'channel': None},
'post': False,
'process': 'channel_names',
'units': 'V'},
'OX_AE': {'desc': 'OX auxiliary electrode raw value',
'id': 'X012',
'kwargs': {'channel': None},
'post': False,
'process': 'channel_names',
'units': 'V'},
'CO': {'desc': 'Calculation of CO based on AAN 803-04',
'id': 152,
'kwargs': {'ae': None,
'alphasense_id': None,
'from_date': None,
'timezone': None,
't': 'EC_SENSOR_TEMP',
'to_date': None,
'we': None},
'post': True,
'process': 'alphasense_803_04',
'units': 'ppb'},
'NO2': {'desc': 'Calculation of NO2 based on AAN 803-04',
'id': 153,
'kwargs': {'ae': None,
'alphasense_id': None,
'from_date': None,
'timezone': None,
't': 'EC_SENSOR_TEMP',
'to_date': None,
'we': None},
'post': True,
'process': 'alphasense_803_04',
'units': 'ppb'},
'O3': {'desc': 'Calculation of O3 based on AAN 803-04',
'id': 157,
'kwargs': {'ae': None,
'alphasense_id': None,
'from_date': None,
'timezone': None,
't': 'EC_SENSOR_TEMP',
'to_date': None,
'we': None},
'post': True,
'process': 'alphasense_803_04',
'units': 'ppb'},
'SO2': {'desc': 'Calculation of SO2 based on AAN 803-04',
'id': 155,
'kwargs': {'ae': None,
'alphasense_id': None,
'from_date': None,
'timezone': None,
't': 'EC_SENSOR_TEMP',
'to_date': None,
'we': None,
'use_alternative': True},
'post': True,
'process': 'alphasense_803_04',
'units': 'ppb'},
'NO': {'desc': 'Calculation of NO based on AAN 803-04',
'id': 154,
'kwargs': {'ae': None,
'alphasense_id': None,
'from_date': None,
'timezone': None,
't': 'EC_SENSOR_TEMP',
'to_date': None,
'we': None},
'post': True,
'process': 'alphasense_803_04',
'units': 'ppb'},
'H2S': {'desc': 'Calculation of H2S based on AAN 803-04',
'id': 156,
'kwargs': {'ae': None,
'alphasense_id': None,
'from_date': None,
'timezone': None,
't': 'EC_SENSOR_TEMP',
'to_date': None,
'we': None},
'post': True,
'process': 'alphasense_803_04',
'units': 'ppb'}},
'min_date': '2021-10-15',
'platform_id': 28,
'processed_data_file': None,
'raw_data_file': None,
'sensors': {'BATT': {'desc': 'Device battery status',
'id': '10',
'units': '%'},
'CCS811_ECO2': {'desc': 'Equivalent CO2', 'id': '112', 'units': 'ppm'},
'CCS811_VOCS': {'desc': 'total Volatile Organics Compounds',
'id': '113',
'units': 'ppm'},
'HUM': {'desc': 'Urban board humidity (SHT31)',
'id': '56',
'units': '%rh'},
'LIGHT': {'desc': 'Urban board ambient light', 'id': '14', 'units': 'lux'},
'NOISE_A': {'desc': 'A-scale noise SPL', 'id': '53', 'units': 'dBA'},
'PRESS': {'desc': 'Atmospheric pressure', 'id': '58', 'units': 'kPa'},
'TEMP': {'desc': 'Urban board temperature (SHT31)',
'id': '55',
'units': 'degC'}},
'source': 'api',
'sources': {'api': {'handler': 'ScApiDevice'},
'csv': {'header_skip': [1, 2, 3],
'index': 'TIME',
'sep': ',',
'tz-aware': True}},
'version': None,
'forwarding_request': None,
'hardware_description': None,
'location': {'longitude': -103.381798267365,
'latitude': 20.7391780551767,
'altitude': 1567}},
'14735': {'skip_blueprint': True,
'blueprint': 'sck_21',
'clean_na': None,
'documentation': 'https://docs.smartcitizen.me/',
'frequency': None,
'id': '14735',
'info_data_file': None,
'timezone': 'Europe/Oslo',
'max_date': '2022-02-15',
'metrics': {'CCS811_ECO2_CLEAN': {'desc': 'eCO2 cleaned data',
'kwargs': {'limits': [400, 65000],
'name': 'CCS811_ECO2',
'window_size': 5,
'window_type': None},
'process': 'clean_ts',
'units': 'ppm',
'post': False,
'id': None},
'CCS811_VOCS_CLEAN': {'desc': 'Volatile Organic Compounds cleaned data',
'kwargs': {'limits': [0, 65000],
'name': 'CCS811_VOCS',
'window_size': 5,
'window_type': None},
'process': 'clean_ts',
'units': 'ppb',
'post': False,
'id': None},
'PM_10_CLEAN': {'desc': 'PM10 calculated based on both PMS5003 PM10 inputs',
'kwargs': {'limits': [0, 1000],
'name': 'PM_10',
'window_size': 5,
'window_type': None},
'process': 'clean_ts',
'units': 'ug/m3',
'post': False,
'id': None},
'PM_1_CLEAN': {'desc': 'PM1 calculated based on both PMS5003 PM10 inputs',
'kwargs': {'limits': [0, 1000],
'name': 'PM_1',
'window_size': 5,
'window_type': None},
'process': 'clean_ts',
'units': 'ug/m3',
'post': False,
'id': None},
'PM_25_CLEAN': {'desc': 'PM2.5 calculated based on both PMS5003 PM2.5 inputs',
'kwargs': {'limits': [0, 1000],
'name': 'PM_25',
'window_size': 5,
'window_type': None},
'process': 'clean_ts',
'units': 'ug/m3',
'post': False,
'id': None}},
'min_date': None,
'platform_id': 28,
'processed_data_file': None,
'raw_data_file': None,
'sensors': {'BATT': {'desc': 'Device battery status',
'id': '10',
'units': '%'},
'CCS811_ECO2': {'desc': 'Equivalent CO2', 'id': '112', 'units': 'ppm'},
'CCS811_VOCS': {'desc': 'total Volatile Organics Compounds',
'id': '113',
'units': 'ppb'},
'HUM': {'desc': 'Urban board humidity (SHT31)',
'id': '56',
'units': '%rh'},
'LIGHT': {'desc': 'Urban board ambient light', 'id': '14', 'units': 'lux'},
'NOISE_A': {'desc': 'A-scale noise SPL', 'id': '53', 'units': 'dBA'},
'PM_1': {'desc': 'PM1 measurement from PMS5003',
'id': '89',
'units': 'ug/m3'},
'PM_10': {'desc': 'PM10 measurement from PMS5003',
'id': '88',
'units': 'ug/m3'},
'PM_25': {'desc': 'PM2.5 measurement from PMS5003',
'id': '87',
'units': 'ug/m3'},
'PRESS': {'desc': 'Atmospheric pressure', 'id': '58', 'units': 'kPa'},
'TEMP': {'desc': 'Urban board temperature (SHT31)',
'id': '55',
'units': 'degC'}},
'source': 'api',
'sources': {'api': {'handler': 'ScApiDevice'},
'csv': {'header_skip': [1, 2, 3],
'index': 'TIME',
'sep': ',',
'tz-aware': True}},
'version': None,
'forwarding_request': 'nilu',
'hardware_description': {'blueprint_url': 'https://raw.githubusercontent.com/fablabbcn/smartcitizen-data/master/blueprints/sck_21.json',
'description': 'Smart Citizen Kit 2.1 With PMS5003',
'versions': [],
'forwarding': 'nilu'},
'location': {'longitude': 11.05276,
'latitude': 59.975056,
'altitude': 136}},
'csv_device': {'skip_blueprint': True,
'blueprint': 'sck_21',
'clean_na': None,
'documentation': 'https://docs.smartcitizen.me/',
'frequency': '1Min',
'id': 'csv_device',
'info_data_file': None,
'timezone': 'Europe/Madrid',
'max_date': None,
'metrics': {'CCS811_ECO2_CLEAN': {'desc': 'eCO2 cleaned data',
'kwargs': {'limits': [400, 65000],
'name': 'CCS811_ECO2',
'window_size': 5,
'window_type': None},
'process': 'clean_ts',
'units': 'ppm',
'post': False,
'id': None},
'CCS811_VOCS_CLEAN': {'desc': 'Volatile Organic Compounds cleaned data',
'kwargs': {'limits': [0, 65000],
'name': 'CCS811_VOCS',
'window_size': 5,
'window_type': None},
'process': 'clean_ts',
'units': 'ppb',
'post': False,
'id': None},
'PM_10_CLEAN': {'desc': 'PM10 calculated based on both PMS5003 PM10 inputs',
'kwargs': {'limits': [0, 1000],
'name': 'PM_10',
'window_size': 5,
'window_type': None},
'process': 'clean_ts',
'units': 'ug/m3',
'post': False,
'id': None},
'PM_1_CLEAN': {'desc': 'PM1 calculated based on both PMS5003 PM10 inputs',
'kwargs': {'limits': [0, 1000],
'name': 'PM_1',
'window_size': 5,
'window_type': None},
'process': 'clean_ts',
'units': 'ug/m3',
'post': False,
'id': None},
'PM_25_CLEAN': {'desc': 'PM2.5 calculated based on both PMS5003 PM2.5 inputs',
'kwargs': {'limits': [0, 1000],
'name': 'PM_25',
'window_size': 5,
'window_type': None},
'process': 'clean_ts',
'units': 'ug/m3',
'post': False,
'id': None}},
'min_date': None,
'platform_id': 28,
'processed_data_file': '2022_05_EXAMPLE_MINKE_csv_device.csv',
'raw_data_file': 'example.csv',
'sensors': {'BATT': {'desc': 'Device battery status',
'id': '10',
'units': '%'},
'CCS811_ECO2': {'desc': 'Equivalent CO2', 'id': '112', 'units': 'ppm'},
'CCS811_VOCS': {'desc': 'total Volatile Organics Compounds',
'id': '113',
'units': 'ppb'},
'HUM': {'desc': 'Urban board humidity (SHT31)',
'id': '56',
'units': '%rh'},
'LIGHT': {'desc': 'Urban board ambient light', 'id': '14', 'units': 'lux'},
'NOISE_A': {'desc': 'A-scale noise SPL', 'id': '53', 'units': 'dBA'},
'PM_1': {'desc': 'PM1 measurement from PMS5003',
'id': '89',
'units': 'ug/m3'},
'PM_10': {'desc': 'PM10 measurement from PMS5003',
'id': '88',
'units': 'ug/m3'},
'PM_25': {'desc': 'PM2.5 measurement from PMS5003',
'id': '87',
'units': 'ug/m3'},
'PRESS': {'desc': 'Atmospheric pressure', 'id': '58', 'units': 'kPa'},
'TEMP': {'desc': 'Urban board temperature (SHT31)',
'id': '55',
'units': 'degC'}},
'source': 'csv',
'sources': {'api': {'handler': 'ScApiDevice'},
'csv': {'header_skip': [1, 2, 3],
'index': 'TIME',
'sep': ',',
'tz-aware': True}},
'version': None,
'forwarding_request': None,
'hardware_description': None}},
'id': '2022_05_EXAMPLE_MINKE',
'notes': '',
'project': '',
'report': '',
'type_test': ''}
Exporting data¶
Finally, we can export the whole test as above (by default in the test directory) or just a device in a defined path
test.devices['14638'].export(path ='~/Desktop')
[2022-05-30 12:26:09] - [SUCCESS] File saved to:
~/Desktop/14638.csv
True
Export the data and test descriptor file
test.to_html(title='Example');
Or save the whole test in the default test processed folder with test.to_csv()